2024-11-11 07:29:08,292 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 07:29:08,308 main DEBUG Took 0.013094 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 07:29:08,308 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 07:29:08,309 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 07:29:08,310 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 07:29:08,312 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:29:08,330 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 07:29:08,349 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:29:08,351 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:29:08,352 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:29:08,352 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:29:08,353 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:29:08,354 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:29:08,355 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:29:08,355 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:29:08,356 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:29:08,356 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:29:08,357 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:29:08,358 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:29:08,358 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:29:08,359 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:29:08,360 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:29:08,360 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:29:08,361 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:29:08,361 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:29:08,362 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:29:08,362 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:29:08,363 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:29:08,363 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:29:08,364 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:29:08,364 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 07:29:08,364 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:29:08,365 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 07:29:08,367 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 07:29:08,369 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 07:29:08,371 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 07:29:08,372 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 07:29:08,373 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 07:29:08,374 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 07:29:08,384 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 07:29:08,387 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 07:29:08,389 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 07:29:08,390 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 07:29:08,390 main DEBUG createAppenders(={Console}) 2024-11-11 07:29:08,392 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-11 07:29:08,393 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-11 07:29:08,393 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-11 07:29:08,394 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 07:29:08,394 main DEBUG OutputStream closed 2024-11-11 07:29:08,395 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 07:29:08,395 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 07:29:08,395 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-11 07:29:08,495 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 07:29:08,498 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 07:29:08,499 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 07:29:08,501 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 07:29:08,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 07:29:08,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 07:29:08,502 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 07:29:08,503 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 07:29:08,503 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 07:29:08,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 07:29:08,504 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 07:29:08,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 07:29:08,505 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 07:29:08,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 07:29:08,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 07:29:08,506 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 07:29:08,507 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 07:29:08,508 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 07:29:08,511 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 07:29:08,512 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-11 07:29:08,512 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 07:29:08,513 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-11T07:29:08,930 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0 2024-11-11 07:29:08,935 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 07:29:08,935 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T07:29:08,947 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-11T07:29:08,998 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T07:29:09,019 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/cluster_43578dcd-0ad0-2e50-c990-4483826cf543, deleteOnExit=true 2024-11-11T07:29:09,020 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-11T07:29:09,021 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/test.cache.data in system properties and HBase conf 2024-11-11T07:29:09,022 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T07:29:09,023 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/hadoop.log.dir in system properties and HBase conf 2024-11-11T07:29:09,024 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T07:29:09,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T07:29:09,025 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-11T07:29:09,137 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T07:29:09,262 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T07:29:09,271 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T07:29:09,272 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T07:29:09,274 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T07:29:09,275 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T07:29:09,276 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T07:29:09,278 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T07:29:09,285 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T07:29:09,286 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T07:29:09,291 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T07:29:09,292 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/nfs.dump.dir in system properties and HBase conf 2024-11-11T07:29:09,292 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/java.io.tmpdir in system properties and HBase conf 2024-11-11T07:29:09,293 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T07:29:09,293 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T07:29:09,294 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T07:29:10,582 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T07:29:10,688 INFO [Time-limited test {}] log.Log(170): Logging initialized @3274ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T07:29:10,796 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T07:29:10,895 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T07:29:10,935 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T07:29:10,935 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T07:29:10,937 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T07:29:10,956 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T07:29:10,960 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/hadoop.log.dir/,AVAILABLE} 2024-11-11T07:29:10,962 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T07:29:11,215 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4ded99d3{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/java.io.tmpdir/jetty-localhost-43731-hadoop-hdfs-3_4_1-tests_jar-_-any-16764782685431111880/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T07:29:11,223 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:43731} 2024-11-11T07:29:11,224 INFO [Time-limited test {}] server.Server(415): Started @3810ms 2024-11-11T07:29:11,863 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T07:29:11,873 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T07:29:11,881 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T07:29:11,881 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T07:29:11,882 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T07:29:11,883 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/hadoop.log.dir/,AVAILABLE} 2024-11-11T07:29:11,883 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T07:29:12,031 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4269bd31{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/java.io.tmpdir/jetty-localhost-33163-hadoop-hdfs-3_4_1-tests_jar-_-any-11776739984957266637/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T07:29:12,033 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:33163} 2024-11-11T07:29:12,033 INFO [Time-limited test {}] server.Server(415): Started @4620ms 2024-11-11T07:29:12,101 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T07:29:13,180 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/cluster_43578dcd-0ad0-2e50-c990-4483826cf543/data/data2/current/BP-1596547718-172.17.0.2-1731310150087/current, will proceed with Du for space computation calculation, 2024-11-11T07:29:13,180 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/cluster_43578dcd-0ad0-2e50-c990-4483826cf543/data/data1/current/BP-1596547718-172.17.0.2-1731310150087/current, will proceed with Du for space computation calculation, 2024-11-11T07:29:13,226 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T07:29:13,284 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfd02afa4ff285579 with lease ID 0xd698546211cb91a: Processing first storage report for DS-ed4b9a31-ba4c-4a33-a958-3a28a3ad81ea from datanode DatanodeRegistration(127.0.0.1:34017, datanodeUuid=19ca5789-f699-41a7-beb6-23556689d733, infoPort=41447, infoSecurePort=0, ipcPort=35103, storageInfo=lv=-57;cid=testClusterID;nsid=1470583051;c=1731310150087) 2024-11-11T07:29:13,287 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfd02afa4ff285579 with lease ID 0xd698546211cb91a: from storage DS-ed4b9a31-ba4c-4a33-a958-3a28a3ad81ea node DatanodeRegistration(127.0.0.1:34017, datanodeUuid=19ca5789-f699-41a7-beb6-23556689d733, infoPort=41447, infoSecurePort=0, ipcPort=35103, storageInfo=lv=-57;cid=testClusterID;nsid=1470583051;c=1731310150087), blocks: 0, hasStaleStorage: true, processing time: 3 msecs, invalidatedBlocks: 0 2024-11-11T07:29:13,287 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfd02afa4ff285579 with lease ID 0xd698546211cb91a: Processing first storage report for DS-f79312a0-22d2-4f12-aae8-9e27fb49f801 from datanode DatanodeRegistration(127.0.0.1:34017, datanodeUuid=19ca5789-f699-41a7-beb6-23556689d733, infoPort=41447, infoSecurePort=0, ipcPort=35103, storageInfo=lv=-57;cid=testClusterID;nsid=1470583051;c=1731310150087) 2024-11-11T07:29:13,288 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfd02afa4ff285579 with lease ID 0xd698546211cb91a: from storage DS-f79312a0-22d2-4f12-aae8-9e27fb49f801 node DatanodeRegistration(127.0.0.1:34017, datanodeUuid=19ca5789-f699-41a7-beb6-23556689d733, infoPort=41447, infoSecurePort=0, ipcPort=35103, storageInfo=lv=-57;cid=testClusterID;nsid=1470583051;c=1731310150087), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T07:29:13,338 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0 2024-11-11T07:29:13,432 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/cluster_43578dcd-0ad0-2e50-c990-4483826cf543/zookeeper_0, clientPort=52366, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/cluster_43578dcd-0ad0-2e50-c990-4483826cf543/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/cluster_43578dcd-0ad0-2e50-c990-4483826cf543/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T07:29:13,460 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=52366 2024-11-11T07:29:13,475 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T07:29:13,478 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T07:29:13,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741825_1001 (size=7) 2024-11-11T07:29:14,203 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e with version=8 2024-11-11T07:29:14,203 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/hbase-staging 2024-11-11T07:29:14,324 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T07:29:14,649 INFO [Time-limited test {}] client.ConnectionUtils(128): master/ce2cf0244921:0 server-side Connection retries=45 2024-11-11T07:29:14,661 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T07:29:14,661 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T07:29:14,667 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T07:29:14,667 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T07:29:14,668 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T07:29:14,837 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-11T07:29:14,894 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T07:29:14,909 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T07:29:14,914 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T07:29:14,945 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 34519 (auto-detected) 2024-11-11T07:29:14,946 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-11T07:29:14,976 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33029 2024-11-11T07:29:15,006 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33029 connecting to ZooKeeper ensemble=127.0.0.1:52366 2024-11-11T07:29:15,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:330290x0, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T07:29:15,126 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33029-0x1012a1601840000 connected 2024-11-11T07:29:15,217 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T07:29:15,220 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T07:29:15,235 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T07:29:15,238 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e, hbase.cluster.distributed=false 2024-11-11T07:29:15,284 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T07:29:15,290 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33029 2024-11-11T07:29:15,290 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33029 2024-11-11T07:29:15,292 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33029 2024-11-11T07:29:15,294 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33029 2024-11-11T07:29:15,294 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33029 2024-11-11T07:29:15,405 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/ce2cf0244921:0 server-side Connection retries=45 2024-11-11T07:29:15,406 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T07:29:15,406 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T07:29:15,407 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T07:29:15,407 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T07:29:15,407 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T07:29:15,409 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T07:29:15,412 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T07:29:15,413 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33345 2024-11-11T07:29:15,415 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33345 connecting to ZooKeeper ensemble=127.0.0.1:52366 2024-11-11T07:29:15,417 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T07:29:15,420 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T07:29:15,454 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:333450x0, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T07:29:15,455 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:333450x0, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T07:29:15,455 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33345-0x1012a1601840001 connected 2024-11-11T07:29:15,460 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T07:29:15,469 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T07:29:15,472 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T07:29:15,481 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T07:29:15,483 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33345 2024-11-11T07:29:15,486 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33345 2024-11-11T07:29:15,487 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33345 2024-11-11T07:29:15,490 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33345 2024-11-11T07:29:15,491 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33345 2024-11-11T07:29:15,512 DEBUG [M:0;ce2cf0244921:33029 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;ce2cf0244921:33029 2024-11-11T07:29:15,513 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/ce2cf0244921,33029,1731310154427 2024-11-11T07:29:15,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T07:29:15,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T07:29:15,542 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/ce2cf0244921,33029,1731310154427 2024-11-11T07:29:15,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T07:29:15,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:29:15,575 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T07:29:15,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:29:15,576 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/ce2cf0244921,33029,1731310154427 from backup master directory 2024-11-11T07:29:15,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/ce2cf0244921,33029,1731310154427 2024-11-11T07:29:15,589 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T07:29:15,590 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T07:29:15,590 WARN [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T07:29:15,591 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=ce2cf0244921,33029,1731310154427 2024-11-11T07:29:15,593 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T07:29:15,595 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T07:29:15,664 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/hbase.id] with ID: a1f130b9-1abe-4225-8bb4-570f54060737 2024-11-11T07:29:15,665 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/.tmp/hbase.id 2024-11-11T07:29:15,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741826_1002 (size=42) 2024-11-11T07:29:16,081 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/.tmp/hbase.id]:[hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/hbase.id] 2024-11-11T07:29:16,133 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T07:29:16,139 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-11T07:29:16,164 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-11-11T07:29:16,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:29:16,220 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:29:16,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741827_1003 (size=196) 2024-11-11T07:29:16,659 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:29:16,663 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T07:29:16,683 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:16,688 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T07:29:16,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741828_1004 (size=1189) 2024-11-11T07:29:16,752 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store 2024-11-11T07:29:16,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741829_1005 (size=34) 2024-11-11T07:29:16,783 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T07:29:16,787 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:29:16,789 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T07:29:16,789 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T07:29:16,789 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T07:29:16,791 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T07:29:16,792 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T07:29:16,792 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T07:29:16,794 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731310156788Disabling compacts and flushes for region at 1731310156788Disabling writes for close at 1731310156791 (+3 ms)Writing region close event to WAL at 1731310156792 (+1 ms)Closed at 1731310156792 2024-11-11T07:29:16,796 WARN [master/ce2cf0244921:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/.initializing 2024-11-11T07:29:16,797 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/WALs/ce2cf0244921,33029,1731310154427 2024-11-11T07:29:16,808 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T07:29:16,828 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ce2cf0244921%2C33029%2C1731310154427, suffix=, logDir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/WALs/ce2cf0244921,33029,1731310154427, archiveDir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/oldWALs, maxLogs=10 2024-11-11T07:29:16,860 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/WALs/ce2cf0244921,33029,1731310154427/ce2cf0244921%2C33029%2C1731310154427.1731310156834, exclude list is [], retry=0 2024-11-11T07:29:16,882 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34017,DS-ed4b9a31-ba4c-4a33-a958-3a28a3ad81ea,DISK] 2024-11-11T07:29:16,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T07:29:16,928 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/WALs/ce2cf0244921,33029,1731310154427/ce2cf0244921%2C33029%2C1731310154427.1731310156834 2024-11-11T07:29:16,930 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41447:41447)] 2024-11-11T07:29:16,930 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:29:16,931 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:29:16,935 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:29:16,937 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:29:16,986 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:29:17,018 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T07:29:17,022 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:17,025 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:29:17,026 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:29:17,031 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T07:29:17,031 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:17,032 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:29:17,033 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:29:17,036 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T07:29:17,037 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:17,038 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:29:17,038 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:29:17,043 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T07:29:17,044 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:17,045 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:29:17,045 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:29:17,050 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:29:17,052 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:29:17,059 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:29:17,060 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:29:17,064 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T07:29:17,069 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T07:29:17,076 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T07:29:17,078 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60063309, jitterRate=-0.10498695075511932}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T07:29:17,086 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731310156953Initializing all the Stores at 1731310156956 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731310156957 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310156958 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310156958Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310156958Cleaning up temporary data from old regions at 1731310157060 (+102 ms)Region opened successfully at 1731310157086 (+26 ms) 2024-11-11T07:29:17,088 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T07:29:17,130 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40db1383, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ce2cf0244921/172.17.0.2:0 2024-11-11T07:29:17,172 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-11T07:29:17,186 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T07:29:17,186 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T07:29:17,190 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T07:29:17,192 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-11T07:29:17,198 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-11T07:29:17,199 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T07:29:17,237 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T07:29:17,248 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T07:29:17,303 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-11T07:29:17,307 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T07:29:17,309 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T07:29:17,320 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-11T07:29:17,323 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T07:29:17,327 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T07:29:17,336 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-11T07:29:17,338 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T07:29:17,347 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T07:29:17,368 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T07:29:17,378 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T07:29:17,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T07:29:17,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T07:29:17,389 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:29:17,390 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:29:17,393 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=ce2cf0244921,33029,1731310154427, sessionid=0x1012a1601840000, setting cluster-up flag (Was=false) 2024-11-11T07:29:17,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:29:17,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:29:17,447 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T07:29:17,450 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ce2cf0244921,33029,1731310154427 2024-11-11T07:29:17,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:29:17,472 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:29:17,498 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T07:29:17,500 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=ce2cf0244921,33029,1731310154427 2024-11-11T07:29:17,507 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-11T07:29:17,597 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(746): ClusterId : a1f130b9-1abe-4225-8bb4-570f54060737 2024-11-11T07:29:17,597 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-11T07:29:17,600 DEBUG [RS:0;ce2cf0244921:33345 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T07:29:17,608 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-11T07:29:17,616 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T07:29:17,622 DEBUG [RS:0;ce2cf0244921:33345 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T07:29:17,622 DEBUG [RS:0;ce2cf0244921:33345 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T07:29:17,622 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: ce2cf0244921,33029,1731310154427 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T07:29:17,632 DEBUG [RS:0;ce2cf0244921:33345 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T07:29:17,633 DEBUG [RS:0;ce2cf0244921:33345 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@284396d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=ce2cf0244921/172.17.0.2:0 2024-11-11T07:29:17,634 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/ce2cf0244921:0, corePoolSize=5, maxPoolSize=5 2024-11-11T07:29:17,634 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/ce2cf0244921:0, corePoolSize=5, maxPoolSize=5 2024-11-11T07:29:17,634 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/ce2cf0244921:0, corePoolSize=5, maxPoolSize=5 2024-11-11T07:29:17,634 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/ce2cf0244921:0, corePoolSize=5, maxPoolSize=5 2024-11-11T07:29:17,634 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/ce2cf0244921:0, corePoolSize=10, maxPoolSize=10 2024-11-11T07:29:17,635 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:29:17,635 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/ce2cf0244921:0, corePoolSize=2, maxPoolSize=2 2024-11-11T07:29:17,635 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:29:17,647 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731310187647 2024-11-11T07:29:17,649 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T07:29:17,651 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T07:29:17,655 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T07:29:17,655 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-11T07:29:17,658 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T07:29:17,658 DEBUG [RS:0;ce2cf0244921:33345 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;ce2cf0244921:33345 2024-11-11T07:29:17,660 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T07:29:17,660 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T07:29:17,661 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T07:29:17,662 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-11T07:29:17,663 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-11T07:29:17,663 DEBUG [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-11T07:29:17,663 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:17,666 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(2659): reportForDuty to master=ce2cf0244921,33029,1731310154427 with port=33345, startcode=1731310155369 2024-11-11T07:29:17,667 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T07:29:17,668 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T07:29:17,669 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T07:29:17,674 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T07:29:17,674 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T07:29:17,676 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:17,676 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T07:29:17,678 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/ce2cf0244921:0:becomeActiveMaster-HFileCleaner.large.0-1731310157676,5,FailOnTimeoutGroup] 2024-11-11T07:29:17,679 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/ce2cf0244921:0:becomeActiveMaster-HFileCleaner.small.0-1731310157678,5,FailOnTimeoutGroup] 2024-11-11T07:29:17,680 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:17,680 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T07:29:17,681 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:17,681 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:17,685 DEBUG [RS:0;ce2cf0244921:33345 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T07:29:17,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741831_1007 (size=1321) 2024-11-11T07:29:17,701 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-11T07:29:17,702 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e 2024-11-11T07:29:17,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741832_1008 (size=32) 2024-11-11T07:29:17,728 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:29:17,734 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51407, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T07:29:17,736 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T07:29:17,740 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T07:29:17,740 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:17,741 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33029 {}] master.ServerManager(363): Checking decommissioned status of RegionServer ce2cf0244921,33345,1731310155369 2024-11-11T07:29:17,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:29:17,742 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T07:29:17,744 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33029 {}] master.ServerManager(517): Registering regionserver=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:17,746 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T07:29:17,746 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:17,748 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:29:17,748 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T07:29:17,753 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T07:29:17,754 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:17,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:29:17,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T07:29:17,760 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T07:29:17,760 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:17,762 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:29:17,762 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T07:29:17,764 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740 2024-11-11T07:29:17,765 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740 2024-11-11T07:29:17,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T07:29:17,769 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T07:29:17,773 DEBUG [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e 2024-11-11T07:29:17,773 DEBUG [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:35677 2024-11-11T07:29:17,773 DEBUG [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-11T07:29:17,775 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:29:17,785 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T07:29:17,792 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T07:29:17,793 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61684594, jitterRate=-0.08082792162895203}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:29:17,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731310157728Initializing all the Stores at 1731310157731 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731310157731Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731310157732 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310157732Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731310157732Cleaning up temporary data from old regions at 1731310157770 (+38 ms)Region opened successfully at 1731310157798 (+28 ms) 2024-11-11T07:29:17,798 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T07:29:17,799 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T07:29:17,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T07:29:17,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T07:29:17,799 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T07:29:17,800 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T07:29:17,800 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731310157798Disabling compacts and flushes for region at 1731310157798Disabling writes for close at 1731310157799 (+1 ms)Writing region close event to WAL at 1731310157800 (+1 ms)Closed at 1731310157800 2024-11-11T07:29:17,804 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T07:29:17,804 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-11T07:29:17,805 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T07:29:17,807 DEBUG [RS:0;ce2cf0244921:33345 {}] zookeeper.ZKUtil(111): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/ce2cf0244921,33345,1731310155369 2024-11-11T07:29:17,807 WARN [RS:0;ce2cf0244921:33345 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T07:29:17,807 INFO [RS:0;ce2cf0244921:33345 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T07:29:17,807 DEBUG [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/WALs/ce2cf0244921,33345,1731310155369 2024-11-11T07:29:17,809 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [ce2cf0244921,33345,1731310155369] 2024-11-11T07:29:17,812 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T07:29:17,823 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T07:29:17,827 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T07:29:17,842 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T07:29:17,862 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T07:29:17,868 INFO [RS:0;ce2cf0244921:33345 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T07:29:17,868 INFO [RS:0;ce2cf0244921:33345 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:17,870 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-11T07:29:17,877 INFO [RS:0;ce2cf0244921:33345 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-11T07:29:17,879 INFO [RS:0;ce2cf0244921:33345 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:17,879 DEBUG [RS:0;ce2cf0244921:33345 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:29:17,879 DEBUG [RS:0;ce2cf0244921:33345 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:29:17,880 DEBUG [RS:0;ce2cf0244921:33345 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:29:17,880 DEBUG [RS:0;ce2cf0244921:33345 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:29:17,880 DEBUG [RS:0;ce2cf0244921:33345 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:29:17,880 DEBUG [RS:0;ce2cf0244921:33345 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/ce2cf0244921:0, corePoolSize=2, maxPoolSize=2 2024-11-11T07:29:17,880 DEBUG [RS:0;ce2cf0244921:33345 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:29:17,880 DEBUG [RS:0;ce2cf0244921:33345 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:29:17,881 DEBUG [RS:0;ce2cf0244921:33345 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:29:17,881 DEBUG [RS:0;ce2cf0244921:33345 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:29:17,881 DEBUG [RS:0;ce2cf0244921:33345 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:29:17,881 DEBUG [RS:0;ce2cf0244921:33345 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/ce2cf0244921:0, corePoolSize=1, maxPoolSize=1 2024-11-11T07:29:17,881 DEBUG [RS:0;ce2cf0244921:33345 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/ce2cf0244921:0, corePoolSize=3, maxPoolSize=3 2024-11-11T07:29:17,882 DEBUG [RS:0;ce2cf0244921:33345 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0, corePoolSize=3, maxPoolSize=3 2024-11-11T07:29:17,883 INFO [RS:0;ce2cf0244921:33345 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:17,883 INFO [RS:0;ce2cf0244921:33345 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:17,884 INFO [RS:0;ce2cf0244921:33345 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:17,884 INFO [RS:0;ce2cf0244921:33345 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:17,884 INFO [RS:0;ce2cf0244921:33345 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:17,884 INFO [RS:0;ce2cf0244921:33345 {}] hbase.ChoreService(168): Chore ScheduledChore name=ce2cf0244921,33345,1731310155369-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T07:29:17,909 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T07:29:17,911 INFO [RS:0;ce2cf0244921:33345 {}] hbase.ChoreService(168): Chore ScheduledChore name=ce2cf0244921,33345,1731310155369-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:17,912 INFO [RS:0;ce2cf0244921:33345 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:17,912 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.Replication(171): ce2cf0244921,33345,1731310155369 started 2024-11-11T07:29:17,936 INFO [RS:0;ce2cf0244921:33345 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:17,936 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(1482): Serving as ce2cf0244921,33345,1731310155369, RpcServer on ce2cf0244921/172.17.0.2:33345, sessionid=0x1012a1601840001 2024-11-11T07:29:17,937 DEBUG [RS:0;ce2cf0244921:33345 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T07:29:17,937 DEBUG [RS:0;ce2cf0244921:33345 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager ce2cf0244921,33345,1731310155369 2024-11-11T07:29:17,938 DEBUG [RS:0;ce2cf0244921:33345 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ce2cf0244921,33345,1731310155369' 2024-11-11T07:29:17,938 DEBUG [RS:0;ce2cf0244921:33345 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T07:29:17,939 DEBUG [RS:0;ce2cf0244921:33345 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T07:29:17,940 DEBUG [RS:0;ce2cf0244921:33345 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T07:29:17,940 DEBUG [RS:0;ce2cf0244921:33345 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T07:29:17,940 DEBUG [RS:0;ce2cf0244921:33345 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager ce2cf0244921,33345,1731310155369 2024-11-11T07:29:17,940 DEBUG [RS:0;ce2cf0244921:33345 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'ce2cf0244921,33345,1731310155369' 2024-11-11T07:29:17,940 DEBUG [RS:0;ce2cf0244921:33345 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T07:29:17,941 DEBUG [RS:0;ce2cf0244921:33345 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T07:29:17,942 DEBUG [RS:0;ce2cf0244921:33345 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T07:29:17,942 INFO [RS:0;ce2cf0244921:33345 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T07:29:17,942 INFO [RS:0;ce2cf0244921:33345 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T07:29:17,978 WARN [ce2cf0244921:33029 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-11T07:29:18,049 INFO [RS:0;ce2cf0244921:33345 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T07:29:18,052 INFO [RS:0;ce2cf0244921:33345 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ce2cf0244921%2C33345%2C1731310155369, suffix=, logDir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/WALs/ce2cf0244921,33345,1731310155369, archiveDir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/oldWALs, maxLogs=32 2024-11-11T07:29:18,075 DEBUG [RS:0;ce2cf0244921:33345 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/WALs/ce2cf0244921,33345,1731310155369/ce2cf0244921%2C33345%2C1731310155369.1731310158056, exclude list is [], retry=0 2024-11-11T07:29:18,080 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34017,DS-ed4b9a31-ba4c-4a33-a958-3a28a3ad81ea,DISK] 2024-11-11T07:29:18,085 INFO [RS:0;ce2cf0244921:33345 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/WALs/ce2cf0244921,33345,1731310155369/ce2cf0244921%2C33345%2C1731310155369.1731310158056 2024-11-11T07:29:18,085 DEBUG [RS:0;ce2cf0244921:33345 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41447:41447)] 2024-11-11T07:29:18,231 DEBUG [ce2cf0244921:33029 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T07:29:18,245 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:18,253 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ce2cf0244921,33345,1731310155369, state=OPENING 2024-11-11T07:29:18,295 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T07:29:18,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:29:18,306 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:29:18,307 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T07:29:18,308 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T07:29:18,310 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T07:29:18,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:29:18,495 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T07:29:18,499 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34513, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T07:29:18,512 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-11T07:29:18,513 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T07:29:18,513 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T07:29:18,519 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=ce2cf0244921%2C33345%2C1731310155369.meta, suffix=.meta, logDir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/WALs/ce2cf0244921,33345,1731310155369, archiveDir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/oldWALs, maxLogs=32 2024-11-11T07:29:18,540 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(525): When create output stream for /user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/WALs/ce2cf0244921,33345,1731310155369/ce2cf0244921%2C33345%2C1731310155369.meta.1731310158521.meta, exclude list is [], retry=0 2024-11-11T07:29:18,545 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34017,DS-ed4b9a31-ba4c-4a33-a958-3a28a3ad81ea,DISK] 2024-11-11T07:29:18,548 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/WALs/ce2cf0244921,33345,1731310155369/ce2cf0244921%2C33345%2C1731310155369.meta.1731310158521.meta 2024-11-11T07:29:18,549 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41447:41447)] 2024-11-11T07:29:18,550 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:29:18,552 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T07:29:18,555 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T07:29:18,561 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T07:29:18,567 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T07:29:18,568 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:29:18,568 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-11T07:29:18,568 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-11T07:29:18,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T07:29:18,574 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T07:29:18,575 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:18,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:29:18,576 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-11T07:29:18,578 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-11T07:29:18,578 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:18,579 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:29:18,580 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T07:29:18,582 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T07:29:18,582 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:18,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:29:18,583 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T07:29:18,585 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T07:29:18,585 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:18,587 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T07:29:18,587 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-11T07:29:18,589 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740 2024-11-11T07:29:18,592 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740 2024-11-11T07:29:18,595 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-11T07:29:18,596 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-11T07:29:18,597 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:29:18,600 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-11T07:29:18,602 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58970542, jitterRate=-0.12127044796943665}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:29:18,602 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-11T07:29:18,604 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731310158569Writing region info on filesystem at 1731310158569Initializing all the Stores at 1731310158571 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731310158571Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731310158572 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310158572Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731310158572Cleaning up temporary data from old regions at 1731310158596 (+24 ms)Running coprocessor post-open hooks at 1731310158603 (+7 ms)Region opened successfully at 1731310158604 (+1 ms) 2024-11-11T07:29:18,613 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731310158484 2024-11-11T07:29:18,627 DEBUG [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T07:29:18,627 INFO [RS_OPEN_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-11T07:29:18,629 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:18,632 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as ce2cf0244921,33345,1731310155369, state=OPEN 2024-11-11T07:29:18,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T07:29:18,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T07:29:18,688 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T07:29:18,688 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T07:29:18,689 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:18,696 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T07:29:18,696 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=ce2cf0244921,33345,1731310155369 in 376 msec 2024-11-11T07:29:18,705 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T07:29:18,706 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 886 msec 2024-11-11T07:29:18,707 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-11T07:29:18,708 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-11T07:29:18,731 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:18,732 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:18,756 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:18,759 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40695, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:18,785 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2380 sec 2024-11-11T07:29:18,785 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731310158785, completionTime=-1 2024-11-11T07:29:18,789 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T07:29:18,789 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-11T07:29:18,862 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-11T07:29:18,862 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731310218862 2024-11-11T07:29:18,862 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731310278862 2024-11-11T07:29:18,862 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 73 msec 2024-11-11T07:29:18,865 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ce2cf0244921,33029,1731310154427-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:18,872 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ce2cf0244921,33029,1731310154427-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:18,873 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ce2cf0244921,33029,1731310154427-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:18,874 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-ce2cf0244921:33029, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:18,875 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:18,876 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:18,885 DEBUG [master/ce2cf0244921:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-11T07:29:18,911 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.319sec 2024-11-11T07:29:18,912 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T07:29:18,913 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T07:29:18,914 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T07:29:18,915 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T07:29:18,915 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T07:29:18,916 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ce2cf0244921,33029,1731310154427-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T07:29:18,916 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ce2cf0244921,33029,1731310154427-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T07:29:18,924 DEBUG [master/ce2cf0244921:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-11T07:29:18,925 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T07:29:18,926 INFO [master/ce2cf0244921:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ce2cf0244921,33029,1731310154427-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T07:29:19,009 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@343aff6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:19,012 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T07:29:19,012 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T07:29:19,016 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:19,020 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:19,027 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:19,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6caff192, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:19,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:19,034 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:19,036 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:19,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32372268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:19,040 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:19,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:19,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:19,071 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=ce2cf0244921,33029,1731310154427 2024-11-11T07:29:19,081 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=750, ProcessCount=11, AvailableMemoryMB=3211 2024-11-11T07:29:19,092 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T07:29:19,098 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is ce2cf0244921,33029,1731310154427 2024-11-11T07:29:19,101 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@63a7af04 2024-11-11T07:29:19,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T07:29:19,105 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T07:29:19,108 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:29:19,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:29:19,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T07:29:19,124 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T07:29:19,126 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-11T07:29:19,127 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:19,130 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T07:29:19,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T07:29:19,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741835_1011 (size=1061) 2024-11-11T07:29:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T07:29:19,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T07:29:19,548 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e 2024-11-11T07:29:19,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741836_1012 (size=53) 2024-11-11T07:29:19,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T07:29:19,961 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:29:19,961 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6756dec3886a521f131ee738c3465dc1, disabling compactions & flushes 2024-11-11T07:29:19,961 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:19,961 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:19,961 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. after waiting 0 ms 2024-11-11T07:29:19,961 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:19,961 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:19,962 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6756dec3886a521f131ee738c3465dc1: Waiting for close lock at 1731310159961Disabling compacts and flushes for region at 1731310159961Disabling writes for close at 1731310159961Writing region close event to WAL at 1731310159961Closed at 1731310159961 2024-11-11T07:29:19,964 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T07:29:19,971 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731310159965"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731310159965"}]},"ts":"1731310159965"} 2024-11-11T07:29:19,977 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T07:29:19,979 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T07:29:19,982 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310159979"}]},"ts":"1731310159979"} 2024-11-11T07:29:19,988 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T07:29:19,990 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6756dec3886a521f131ee738c3465dc1, ASSIGN}] 2024-11-11T07:29:19,993 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6756dec3886a521f131ee738c3465dc1, ASSIGN 2024-11-11T07:29:19,995 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6756dec3886a521f131ee738c3465dc1, ASSIGN; state=OFFLINE, location=ce2cf0244921,33345,1731310155369; forceNewPlan=false, retain=false 2024-11-11T07:29:20,147 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6756dec3886a521f131ee738c3465dc1, regionState=OPENING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:20,152 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6756dec3886a521f131ee738c3465dc1, ASSIGN because future has completed 2024-11-11T07:29:20,154 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:29:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T07:29:20,316 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:20,316 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:29:20,316 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:20,317 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:29:20,317 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:20,317 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:20,320 INFO [StoreOpener-6756dec3886a521f131ee738c3465dc1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:20,323 INFO [StoreOpener-6756dec3886a521f131ee738c3465dc1-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:29:20,324 INFO [StoreOpener-6756dec3886a521f131ee738c3465dc1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6756dec3886a521f131ee738c3465dc1 columnFamilyName A 2024-11-11T07:29:20,324 DEBUG [StoreOpener-6756dec3886a521f131ee738c3465dc1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:20,326 INFO [StoreOpener-6756dec3886a521f131ee738c3465dc1-1 {}] regionserver.HStore(327): Store=6756dec3886a521f131ee738c3465dc1/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:29:20,326 INFO [StoreOpener-6756dec3886a521f131ee738c3465dc1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:20,328 INFO [StoreOpener-6756dec3886a521f131ee738c3465dc1-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:29:20,329 INFO [StoreOpener-6756dec3886a521f131ee738c3465dc1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6756dec3886a521f131ee738c3465dc1 columnFamilyName B 2024-11-11T07:29:20,329 DEBUG [StoreOpener-6756dec3886a521f131ee738c3465dc1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:20,330 INFO [StoreOpener-6756dec3886a521f131ee738c3465dc1-1 {}] regionserver.HStore(327): Store=6756dec3886a521f131ee738c3465dc1/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:29:20,330 INFO [StoreOpener-6756dec3886a521f131ee738c3465dc1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:20,333 INFO [StoreOpener-6756dec3886a521f131ee738c3465dc1-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:29:20,333 INFO [StoreOpener-6756dec3886a521f131ee738c3465dc1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6756dec3886a521f131ee738c3465dc1 columnFamilyName C 2024-11-11T07:29:20,333 DEBUG [StoreOpener-6756dec3886a521f131ee738c3465dc1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:20,337 INFO [StoreOpener-6756dec3886a521f131ee738c3465dc1-1 {}] regionserver.HStore(327): Store=6756dec3886a521f131ee738c3465dc1/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:29:20,338 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:20,338 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:20,340 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:20,341 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:20,344 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:20,344 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:20,346 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:29:20,349 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:20,353 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T07:29:20,355 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6756dec3886a521f131ee738c3465dc1; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60557770, jitterRate=-0.09761890769004822}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:29:20,355 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:20,357 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6756dec3886a521f131ee738c3465dc1: Running coprocessor pre-open hook at 1731310160317Writing region info on filesystem at 1731310160317Initializing all the Stores at 1731310160319 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310160319Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310160319Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310160319Cleaning up temporary data from old regions at 1731310160344 (+25 ms)Running coprocessor post-open hooks at 1731310160355 (+11 ms)Region opened successfully at 1731310160356 (+1 ms) 2024-11-11T07:29:20,360 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., pid=6, masterSystemTime=1731310160309 2024-11-11T07:29:20,365 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:20,365 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:20,367 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6756dec3886a521f131ee738c3465dc1, regionState=OPEN, openSeqNum=2, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:20,372 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:29:20,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T07:29:20,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 in 222 msec 2024-11-11T07:29:20,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T07:29:20,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6756dec3886a521f131ee738c3465dc1, ASSIGN in 392 msec 2024-11-11T07:29:20,394 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T07:29:20,394 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310160394"}]},"ts":"1731310160394"} 2024-11-11T07:29:20,399 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T07:29:20,402 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T07:29:20,409 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2870 sec 2024-11-11T07:29:21,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-11T07:29:21,287 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:21,293 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ad452c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,293 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:21,293 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:21,295 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:21,296 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68edb023, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:21,297 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:21,298 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:21,300 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:21,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6eb2f0c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,302 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:21,304 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:21,305 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:21,308 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34296, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:21,311 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T07:29:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ce2cf0244921,33029,1731310154427 2024-11-11T07:29:21,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f86aa4f 2024-11-11T07:29:21,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T07:29:21,319 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46278, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T07:29:21,325 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19d419ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,325 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:21,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:21,327 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:21,327 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fa64271, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:21,328 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:21,330 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:21,332 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:21,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@27108bea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,335 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@130d4a2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,335 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:21,336 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:21,337 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:21,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b0e6697, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:21,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:21,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:21,346 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:21,348 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab1aa4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,350 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59d35085, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,350 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:21,350 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:21,354 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:21,354 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:21,355 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b0e30d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:21,356 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:21,359 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:21,360 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:21,364 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60bd1ae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fb4b46b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,367 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:21,367 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:21,370 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:21,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54ede97e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:21,371 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:21,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:21,380 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:21,382 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6974f6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,385 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a1cce95, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,385 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:21,386 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:21,387 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:21,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:21,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@577d1293, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:21,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:21,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:21,390 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46358, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:21,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a9c1ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,399 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3055dae7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,399 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:21,399 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:21,401 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:21,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:21,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1dfb2ab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:21,403 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:21,405 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:21,407 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ce3cf61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,408 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5238acdf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,408 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:21,408 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:21,409 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:21,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:21,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4707cca9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:21,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:21,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:21,413 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46396, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:21,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4934d870, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,420 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f4f2c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,420 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:21,421 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:21,422 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:21,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:21,423 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:21,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b19c399, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:21,424 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:21,425 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:21,426 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a18c602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,431 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6643136e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,431 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:21,431 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:21,433 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:21,433 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:21,433 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:21,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@159e37c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:21,434 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:21,435 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:21,436 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:21,437 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72ad5b9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:21,449 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:21,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:21,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:21,455 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:21,459 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:21,462 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:21,463 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:21,465 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:21,466 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:29:21,468 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:21,469 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:21,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:29:21,471 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:21,471 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34316, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:21,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:21,473 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:21,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34320, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:21,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:29:21,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:29:21,501 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:21,510 DEBUG [Thread-143 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:21,511 DEBUG [Thread-143 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:21,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:21,517 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:29:21,525 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:21,531 DEBUG [Thread-160 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:21,531 DEBUG [Thread-160 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:21,533 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34334, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:29:21,543 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:21,545 DEBUG [Thread-154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:21,546 DEBUG [Thread-154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:21,549 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34350, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:21,554 DEBUG [Thread-154 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:29:21,556 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:21,561 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:21,562 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:21,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:21,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:21,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:21,567 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34362, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:21,572 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:21,574 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:29:21,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:29:21,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:21,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T07:29:21,595 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:29:21,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-11T07:29:21,607 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:29:21,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:21,609 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:29:21,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:21,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:21,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:21,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:21,610 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:21,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T07:29:21,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:29:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T07:29:21,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/dd5b766af4f246b8a140d6ef38ac589b is 50, key is test_row_1/A:col10/1731310161585/Put/seqid=0 2024-11-11T07:29:21,818 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:29:21,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:21,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:21,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:21,830 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:21,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:21,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:21,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741837_1013 (size=16677) 2024-11-11T07:29:21,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T07:29:21,936 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731310161680, max timestamp=1731310161911Num uniques 859; 2024-11-11T07:29:21,937 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T07:29:21,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310161680, max timestamp=1731310161911Num uniques 859; 2024-11-11T07:29:21,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:21,993 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310161680, max timestamp=1731310161922Num uniques 859; 2024-11-11T07:29:21,994 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:22,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:29:22,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:22,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:22,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:22,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:22,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:22,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:22,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:22,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310222071, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:22,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:22,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310222072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:22,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:22,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310222076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:22,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:22,127 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:22,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310222099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:22,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310222084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:22,170 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,170 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,170 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:22,171 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:22,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:22,174 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,175 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:22,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:29:22,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:22,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:22,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:22,191 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:22,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:22,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:22,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T07:29:22,278 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/dd5b766af4f246b8a140d6ef38ac589b 2024-11-11T07:29:22,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:22,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:22,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310222284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:22,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:22,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:22,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310222287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:22,304 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:22,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310222287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310222280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:22,307 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,309 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:22,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310222286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:22,350 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:29:22,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:22,351 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:22,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:22,512 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/c0e1dae6f0774568ab0eb28f8629613a is 50, key is test_row_1/B:col10/1731310161585/Put/seqid=0 2024-11-11T07:29:22,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:29:22,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:22,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:22,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:22,513 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:22,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:22,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:22,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:22,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310222516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:22,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:22,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310222517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:22,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:22,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:22,526 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:22,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310222519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,527 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:22,537 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310222526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:22,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310222517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:22,540 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:22,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741838_1014 (size=9657) 2024-11-11T07:29:22,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/c0e1dae6f0774568ab0eb28f8629613a 2024-11-11T07:29:22,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/5f6d1b1486b148a184f1f7ffb67fe132 is 50, key is test_row_1/C:col10/1731310161585/Put/seqid=0 2024-11-11T07:29:22,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741839_1015 (size=9657) 2024-11-11T07:29:22,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/5f6d1b1486b148a184f1f7ffb67fe132 2024-11-11T07:29:22,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:29:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:22,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:22,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:22,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:22,686 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/dd5b766af4f246b8a140d6ef38ac589b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/dd5b766af4f246b8a140d6ef38ac589b 2024-11-11T07:29:22,707 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/dd5b766af4f246b8a140d6ef38ac589b, entries=250, sequenceid=12, filesize=16.3 K 2024-11-11T07:29:22,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/c0e1dae6f0774568ab0eb28f8629613a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/c0e1dae6f0774568ab0eb28f8629613a 2024-11-11T07:29:22,739 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/c0e1dae6f0774568ab0eb28f8629613a, entries=100, sequenceid=12, filesize=9.4 K 2024-11-11T07:29:22,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/5f6d1b1486b148a184f1f7ffb67fe132 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/5f6d1b1486b148a184f1f7ffb67fe132 2024-11-11T07:29:22,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T07:29:22,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/5f6d1b1486b148a184f1f7ffb67fe132, entries=100, sequenceid=12, filesize=9.4 K 2024-11-11T07:29:22,779 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=129.71 KB/132820 for 6756dec3886a521f131ee738c3465dc1 in 1186ms, sequenceid=12, compaction requested=false 2024-11-11T07:29:22,787 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T07:29:22,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:22,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-11T07:29:22,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:22,833 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=129.71 KB heapSize=370.14 KB 2024-11-11T07:29:22,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:22,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:22,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:22,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:22,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:22,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:22,876 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:22,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/0c0583b3a8f3421dbc04b1f391213259 is 50, key is test_row_0/A:col10/1731310162066/Put/seqid=0 2024-11-11T07:29:22,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741840_1016 (size=12001) 2024-11-11T07:29:22,955 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/0c0583b3a8f3421dbc04b1f391213259 2024-11-11T07:29:23,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/ff96e4e528fc42cea89f75e9496ed252 is 50, key is test_row_0/B:col10/1731310162066/Put/seqid=0 2024-11-11T07:29:22,996 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310222904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,011 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310222976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,020 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741841_1017 (size=12001) 2024-11-11T07:29:23,058 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/ff96e4e528fc42cea89f75e9496ed252 2024-11-11T07:29:23,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/956eae2b6a1a41608cec60f188170af8 is 50, key is test_row_0/C:col10/1731310162066/Put/seqid=0 2024-11-11T07:29:23,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310223009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,105 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310223020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,106 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,107 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310223022, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310223119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,142 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310223125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,159 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741842_1018 (size=12001) 2024-11-11T07:29:23,162 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/956eae2b6a1a41608cec60f188170af8 2024-11-11T07:29:23,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/0c0583b3a8f3421dbc04b1f391213259 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/0c0583b3a8f3421dbc04b1f391213259 2024-11-11T07:29:23,188 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/0c0583b3a8f3421dbc04b1f391213259, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T07:29:23,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/ff96e4e528fc42cea89f75e9496ed252 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ff96e4e528fc42cea89f75e9496ed252 2024-11-11T07:29:23,213 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ff96e4e528fc42cea89f75e9496ed252, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T07:29:23,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/956eae2b6a1a41608cec60f188170af8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/956eae2b6a1a41608cec60f188170af8 2024-11-11T07:29:23,241 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/956eae2b6a1a41608cec60f188170af8, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T07:29:23,247 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~129.71 KB/132820, heapSize ~370.09 KB/378976, currentSize=60.38 KB/61830 for 6756dec3886a521f131ee738c3465dc1 in 415ms, sequenceid=48, compaction requested=false 2024-11-11T07:29:23,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:23,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:23,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-11T07:29:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-11T07:29:23,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-11T07:29:23,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6420 sec 2024-11-11T07:29:23,265 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.6650 sec 2024-11-11T07:29:23,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:23,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T07:29:23,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:23,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:23,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:23,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:23,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:23,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:23,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/eb707fd6a6e6474faea26229db2d87f9 is 50, key is test_row_0/A:col10/1731310163258/Put/seqid=0 2024-11-11T07:29:23,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741843_1019 (size=12001) 2024-11-11T07:29:23,468 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310223426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,470 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,475 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310223434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,479 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,480 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,488 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310223463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,489 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310223472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,535 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310223475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,537 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,538 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310223576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,584 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310223596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,614 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310223586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,619 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310223645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,648 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310223647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/eb707fd6a6e6474faea26229db2d87f9 2024-11-11T07:29:23,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-11T07:29:23,766 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:23,767 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:29:23,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-11T07:29:23,789 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:29:23,791 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:29:23,791 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:29:23,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T07:29:23,805 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/c289bc3659984f9697d3adedd8b96fe4 is 50, key is test_row_0/B:col10/1731310163258/Put/seqid=0 2024-11-11T07:29:23,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310223795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,822 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,823 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,829 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310223826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,833 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,834 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,835 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310223833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741844_1020 (size=12001) 2024-11-11T07:29:23,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/c289bc3659984f9697d3adedd8b96fe4 2024-11-11T07:29:23,876 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310223856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,877 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,877 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:23,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,878 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310223875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:23,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,878 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:23,879 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:23,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/442c137e72a4432c874cbcb9cbaccda8 is 50, key is test_row_0/C:col10/1731310163258/Put/seqid=0 2024-11-11T07:29:23,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T07:29:23,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T07:29:23,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:23,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:23,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:23,948 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:23,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:23,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:23,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741845_1021 (size=12001) 2024-11-11T07:29:23,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/442c137e72a4432c874cbcb9cbaccda8 2024-11-11T07:29:23,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/eb707fd6a6e6474faea26229db2d87f9 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/eb707fd6a6e6474faea26229db2d87f9 2024-11-11T07:29:23,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/eb707fd6a6e6474faea26229db2d87f9, entries=150, sequenceid=62, filesize=11.7 K 2024-11-11T07:29:23,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/c289bc3659984f9697d3adedd8b96fe4 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/c289bc3659984f9697d3adedd8b96fe4 2024-11-11T07:29:24,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/c289bc3659984f9697d3adedd8b96fe4, entries=150, sequenceid=62, filesize=11.7 K 2024-11-11T07:29:24,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/442c137e72a4432c874cbcb9cbaccda8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/442c137e72a4432c874cbcb9cbaccda8 2024-11-11T07:29:24,020 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/442c137e72a4432c874cbcb9cbaccda8, entries=150, sequenceid=62, filesize=11.7 K 2024-11-11T07:29:24,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=114.05 KB/116790 for 6756dec3886a521f131ee738c3465dc1 in 756ms, sequenceid=62, compaction requested=true 2024-11-11T07:29:24,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:24,033 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:29:24,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:24,037 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:24,041 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:24,043 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:24,044 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/B is initiating minor compaction (all files) 2024-11-11T07:29:24,045 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/B in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:24,045 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/c0e1dae6f0774568ab0eb28f8629613a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ff96e4e528fc42cea89f75e9496ed252, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/c289bc3659984f9697d3adedd8b96fe4] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=32.9 K 2024-11-11T07:29:24,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:29:24,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:24,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:29:24,047 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting c0e1dae6f0774568ab0eb28f8629613a, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731310161559 2024-11-11T07:29:24,048 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting ff96e4e528fc42cea89f75e9496ed252, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731310162049 2024-11-11T07:29:24,046 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:24,049 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting c289bc3659984f9697d3adedd8b96fe4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731310162944 2024-11-11T07:29:24,055 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40679 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:24,055 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/A is initiating minor compaction (all files) 2024-11-11T07:29:24,055 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/A in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:24,055 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/dd5b766af4f246b8a140d6ef38ac589b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/0c0583b3a8f3421dbc04b1f391213259, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/eb707fd6a6e6474faea26229db2d87f9] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=39.7 K 2024-11-11T07:29:24,057 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd5b766af4f246b8a140d6ef38ac589b, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731310161521 2024-11-11T07:29:24,059 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c0583b3a8f3421dbc04b1f391213259, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731310162049 2024-11-11T07:29:24,060 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting eb707fd6a6e6474faea26229db2d87f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731310162944 2024-11-11T07:29:24,089 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T07:29:24,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T07:29:24,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-11T07:29:24,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:24,120 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:29:24,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:24,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:24,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:24,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:24,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:24,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:24,128 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#A#compaction#10 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:24,128 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#B#compaction#9 average throughput is 0.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:24,128 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/347541acc3cb4c36bf671748aa341172 is 50, key is test_row_0/A:col10/1731310163258/Put/seqid=0 2024-11-11T07:29:24,132 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/ac63d3a643144c8bb260c745b9852812 is 50, key is test_row_0/B:col10/1731310163258/Put/seqid=0 2024-11-11T07:29:24,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:24,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/56d8fd37ad974ae18218b2ecb6fa2ed5 is 50, key is test_row_0/A:col10/1731310163410/Put/seqid=0 2024-11-11T07:29:24,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:24,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310224186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310224190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,211 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741847_1023 (size=12104) 2024-11-11T07:29:24,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310224208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,236 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310224208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310224213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,247 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/ac63d3a643144c8bb260c745b9852812 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ac63d3a643144c8bb260c745b9852812 2024-11-11T07:29:24,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741848_1024 (size=12001) 2024-11-11T07:29:24,260 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/56d8fd37ad974ae18218b2ecb6fa2ed5 2024-11-11T07:29:24,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741846_1022 (size=12104) 2024-11-11T07:29:24,297 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/B of 6756dec3886a521f131ee738c3465dc1 into ac63d3a643144c8bb260c745b9852812(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:24,297 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:24,302 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/347541acc3cb4c36bf671748aa341172 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/347541acc3cb4c36bf671748aa341172 2024-11-11T07:29:24,308 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/B, priority=13, startTime=1731310164036; duration=0sec 2024-11-11T07:29:24,308 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:24,309 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:B 2024-11-11T07:29:24,309 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:24,311 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T07:29:24,317 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:24,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/8833d1bd5c8243c1bfbdd50b590ff8b4 is 50, key is test_row_0/B:col10/1731310163410/Put/seqid=0 2024-11-11T07:29:24,317 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/C is initiating minor compaction (all files) 2024-11-11T07:29:24,318 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/C in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:24,318 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/5f6d1b1486b148a184f1f7ffb67fe132, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/956eae2b6a1a41608cec60f188170af8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/442c137e72a4432c874cbcb9cbaccda8] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=32.9 K 2024-11-11T07:29:24,323 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f6d1b1486b148a184f1f7ffb67fe132, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731310161559 2024-11-11T07:29:24,324 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 956eae2b6a1a41608cec60f188170af8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731310162049 2024-11-11T07:29:24,325 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 442c137e72a4432c874cbcb9cbaccda8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731310162944 2024-11-11T07:29:24,326 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/A of 6756dec3886a521f131ee738c3465dc1 into 347541acc3cb4c36bf671748aa341172(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:24,327 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:24,327 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/A, priority=13, startTime=1731310164027; duration=0sec 2024-11-11T07:29:24,328 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:24,328 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:A 2024-11-11T07:29:24,334 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310224316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,335 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310224316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,349 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310224347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310224347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310224348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741849_1025 (size=12001) 2024-11-11T07:29:24,398 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#C#compaction#13 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:24,400 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/8166a7110f0f4a16a99152ea0fa1d27c is 50, key is test_row_0/C:col10/1731310163258/Put/seqid=0 2024-11-11T07:29:24,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T07:29:24,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741850_1026 (size=12104) 2024-11-11T07:29:24,502 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/8166a7110f0f4a16a99152ea0fa1d27c as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/8166a7110f0f4a16a99152ea0fa1d27c 2024-11-11T07:29:24,526 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/C of 6756dec3886a521f131ee738c3465dc1 into 8166a7110f0f4a16a99152ea0fa1d27c(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:24,527 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:24,527 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/C, priority=13, startTime=1731310164046; duration=0sec 2024-11-11T07:29:24,527 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:24,527 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:C 2024-11-11T07:29:24,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310224546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,555 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310224555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,580 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310224576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,582 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,591 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310224577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,593 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,594 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310224577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,595 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,794 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/8833d1bd5c8243c1bfbdd50b590ff8b4 2024-11-11T07:29:24,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/09ed1eb053a64ab4b49bdfb2370686e6 is 50, key is test_row_0/C:col10/1731310163410/Put/seqid=0 2024-11-11T07:29:24,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310224868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,875 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741851_1027 (size=12001) 2024-11-11T07:29:24,877 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/09ed1eb053a64ab4b49bdfb2370686e6 2024-11-11T07:29:24,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/56d8fd37ad974ae18218b2ecb6fa2ed5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/56d8fd37ad974ae18218b2ecb6fa2ed5 2024-11-11T07:29:24,895 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T07:29:24,896 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T07:29:24,898 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T07:29:24,898 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T07:29:24,898 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T07:29:24,898 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T07:29:24,899 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310224887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,900 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T07:29:24,900 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T07:29:24,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310224897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,914 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/56d8fd37ad974ae18218b2ecb6fa2ed5, entries=150, sequenceid=82, filesize=11.7 K 2024-11-11T07:29:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,915 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310224907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/8833d1bd5c8243c1bfbdd50b590ff8b4 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/8833d1bd5c8243c1bfbdd50b590ff8b4 2024-11-11T07:29:24,926 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:24,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310224915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:24,932 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/8833d1bd5c8243c1bfbdd50b590ff8b4, entries=150, sequenceid=82, filesize=11.7 K 2024-11-11T07:29:24,934 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/09ed1eb053a64ab4b49bdfb2370686e6 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/09ed1eb053a64ab4b49bdfb2370686e6 2024-11-11T07:29:24,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:24,936 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:24,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T07:29:24,968 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/09ed1eb053a64ab4b49bdfb2370686e6, entries=150, sequenceid=82, filesize=11.7 K 2024-11-11T07:29:24,971 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6756dec3886a521f131ee738c3465dc1 in 851ms, sequenceid=82, compaction requested=false 2024-11-11T07:29:24,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:24,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:24,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-11T07:29:24,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-11T07:29:24,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-11T07:29:24,988 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1880 sec 2024-11-11T07:29:24,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.2220 sec 2024-11-11T07:29:25,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:25,413 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T07:29:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:25,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:25,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/58c1fd5a7f5b4a49aee087d179c91c9e is 50, key is test_row_0/A:col10/1731310164194/Put/seqid=0 2024-11-11T07:29:25,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741852_1028 (size=12001) 2024-11-11T07:29:25,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/58c1fd5a7f5b4a49aee087d179c91c9e 2024-11-11T07:29:25,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:25,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310225502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:25,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:25,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310225503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:25,526 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/68a694390b394396ac084056f2fbfbdd is 50, key is test_row_0/B:col10/1731310164194/Put/seqid=0 2024-11-11T07:29:25,526 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:25,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310225505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:25,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:25,527 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,528 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:25,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:25,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310225523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:25,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,546 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:25,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:25,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310225519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,550 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:25,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741853_1029 (size=12001) 2024-11-11T07:29:25,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:25,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310225635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:25,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:25,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:25,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310225635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:25,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:25,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:25,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310225636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:25,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:25,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310225655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:25,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:25,663 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:25,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310225655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:25,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,668 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:25,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310225855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:25,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:25,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310225856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,865 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:25,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:25,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310225860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:25,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:25,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310225875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,890 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:25,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:25,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310225878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:25,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:25,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-11T07:29:25,946 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:25,946 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:29:25,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-11T07:29:25,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T07:29:25,960 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:29:25,962 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:29:25,962 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:29:25,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/68a694390b394396ac084056f2fbfbdd 2024-11-11T07:29:26,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/2fccaa737f0f4fbc9dcee45b420d0116 is 50, key is test_row_0/C:col10/1731310164194/Put/seqid=0 2024-11-11T07:29:26,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T07:29:26,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741854_1030 (size=12001) 2024-11-11T07:29:26,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/2fccaa737f0f4fbc9dcee45b420d0116 2024-11-11T07:29:26,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:29:26,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:26,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:26,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:26,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/58c1fd5a7f5b4a49aee087d179c91c9e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/58c1fd5a7f5b4a49aee087d179c91c9e 2024-11-11T07:29:26,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,144 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/58c1fd5a7f5b4a49aee087d179c91c9e, entries=150, sequenceid=98, filesize=11.7 K 2024-11-11T07:29:26,147 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/68a694390b394396ac084056f2fbfbdd as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/68a694390b394396ac084056f2fbfbdd 2024-11-11T07:29:26,160 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/68a694390b394396ac084056f2fbfbdd, entries=150, sequenceid=98, filesize=11.7 K 2024-11-11T07:29:26,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/2fccaa737f0f4fbc9dcee45b420d0116 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/2fccaa737f0f4fbc9dcee45b420d0116 2024-11-11T07:29:26,186 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/2fccaa737f0f4fbc9dcee45b420d0116, entries=150, sequenceid=98, filesize=11.7 K 2024-11-11T07:29:26,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6756dec3886a521f131ee738c3465dc1 in 775ms, sequenceid=98, compaction requested=true 2024-11-11T07:29:26,190 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:26,190 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:26,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:29:26,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:26,192 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:29:26,193 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:26,193 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/A is initiating minor compaction (all files) 2024-11-11T07:29:26,193 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/A in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:26,193 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/347541acc3cb4c36bf671748aa341172, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/56d8fd37ad974ae18218b2ecb6fa2ed5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/58c1fd5a7f5b4a49aee087d179c91c9e] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=35.3 K 2024-11-11T07:29:26,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:26,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:29:26,194 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:29:26,194 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 347541acc3cb4c36bf671748aa341172, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731310162944 2024-11-11T07:29:26,194 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:26,196 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56d8fd37ad974ae18218b2ecb6fa2ed5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731310163410 2024-11-11T07:29:26,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:26,197 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58c1fd5a7f5b4a49aee087d179c91c9e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731310164194 2024-11-11T07:29:26,203 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:26,203 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/B is initiating minor compaction (all files) 2024-11-11T07:29:26,203 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/B in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:26,204 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ac63d3a643144c8bb260c745b9852812, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/8833d1bd5c8243c1bfbdd50b590ff8b4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/68a694390b394396ac084056f2fbfbdd] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=35.3 K 2024-11-11T07:29:26,204 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=114.05 KB heapSize=340.13 KB 2024-11-11T07:29:26,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731310165501, max timestamp=1731310166191Num uniques 859; 2024-11-11T07:29:26,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8102, suffix heap size=117064,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=177 2024-11-11T07:29:26,205 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting ac63d3a643144c8bb260c745b9852812, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731310162944 2024-11-11T07:29:26,206 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 8833d1bd5c8243c1bfbdd50b590ff8b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731310163410 2024-11-11T07:29:26,207 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 68a694390b394396ac084056f2fbfbdd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731310164194 2024-11-11T07:29:26,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310165501, max timestamp=1731310166191Num uniques 859; 2024-11-11T07:29:26,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:26,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:26,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:26,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:26,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:26,223 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:26,224 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:26,243 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#A#compaction#18 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:26,244 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/e01afd75d7bd4433a72186b489374df7 is 50, key is test_row_0/A:col10/1731310164194/Put/seqid=0 2024-11-11T07:29:26,252 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#B#compaction#19 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:26,253 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/2501523d99f9496b8d7b32b9c6893e65 is 50, key is test_row_0/B:col10/1731310164194/Put/seqid=0 2024-11-11T07:29:26,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/2630135164e74bb19d74239b7bfffc34 is 50, key is test_row_0/A:col10/1731310166191/Put/seqid=0 2024-11-11T07:29:26,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T07:29:26,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:29:26,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:26,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:26,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:26,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741855_1031 (size=12207) 2024-11-11T07:29:26,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741856_1032 (size=12207) 2024-11-11T07:29:26,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741857_1033 (size=12001) 2024-11-11T07:29:26,364 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310166312, max timestamp=1731310166340Num uniques 859; 2024-11-11T07:29:26,365 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:26,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731310166312, max timestamp=1731310166337Num uniques 859; 2024-11-11T07:29:26,377 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T07:29:26,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:26,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310226394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:26,433 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:26,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,433 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:26,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310226399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:26,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:26,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310226392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,435 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:26,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310226396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:26,436 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310226414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,437 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:26,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:29:26,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:26,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:26,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:26,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:26,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310226545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:26,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:26,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,554 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310226546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:26,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,554 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:26,554 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:26,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310226547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:26,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:26,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:26,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310226563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:26,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310226549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:26,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,581 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:26,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T07:29:26,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:29:26,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:26,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:26,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:26,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,720 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/e01afd75d7bd4433a72186b489374df7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/e01afd75d7bd4433a72186b489374df7 2024-11-11T07:29:26,732 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/A of 6756dec3886a521f131ee738c3465dc1 into e01afd75d7bd4433a72186b489374df7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:26,732 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:26,733 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/A, priority=13, startTime=1731310166190; duration=0sec 2024-11-11T07:29:26,733 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:26,733 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:A 2024-11-11T07:29:26,733 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:26,735 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:26,735 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/C is initiating minor compaction (all files) 2024-11-11T07:29:26,735 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/C in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:26,736 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/8166a7110f0f4a16a99152ea0fa1d27c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/09ed1eb053a64ab4b49bdfb2370686e6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/2fccaa737f0f4fbc9dcee45b420d0116] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=35.3 K 2024-11-11T07:29:26,740 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8166a7110f0f4a16a99152ea0fa1d27c, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731310162944 2024-11-11T07:29:26,741 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09ed1eb053a64ab4b49bdfb2370686e6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731310163410 2024-11-11T07:29:26,742 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=9.75 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/2630135164e74bb19d74239b7bfffc34 2024-11-11T07:29:26,743 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fccaa737f0f4fbc9dcee45b420d0116, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731310164194 2024-11-11T07:29:26,761 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/2501523d99f9496b8d7b32b9c6893e65 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/2501523d99f9496b8d7b32b9c6893e65 2024-11-11T07:29:26,784 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:29:26,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:26,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:26,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:26,785 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,785 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#C#compaction#21 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:26,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:26,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310226765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:26,786 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/665866aabf8243fa8d7223919a804c31 is 50, key is test_row_0/C:col10/1731310164194/Put/seqid=0 2024-11-11T07:29:26,786 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:26,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,788 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:26,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310226765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,789 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310226766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:26,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/4dc0978f596d4cdfbc97943f653b6cfe is 50, key is test_row_0/B:col10/1731310166191/Put/seqid=0 2024-11-11T07:29:26,810 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/B of 6756dec3886a521f131ee738c3465dc1 into 2501523d99f9496b8d7b32b9c6893e65(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:26,810 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:26,810 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/B, priority=13, startTime=1731310166192; duration=0sec 2024-11-11T07:29:26,811 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:26,811 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:B 2024-11-11T07:29:26,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:26,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310226786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:26,815 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:26,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310226791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:26,823 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:26,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741858_1034 (size=12207) 2024-11-11T07:29:26,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741859_1035 (size=12001) 2024-11-11T07:29:26,883 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/665866aabf8243fa8d7223919a804c31 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/665866aabf8243fa8d7223919a804c31 2024-11-11T07:29:26,901 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/C of 6756dec3886a521f131ee738c3465dc1 into 665866aabf8243fa8d7223919a804c31(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:26,901 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:26,901 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/C, priority=13, startTime=1731310166194; duration=0sec 2024-11-11T07:29:26,901 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:26,901 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:C 2024-11-11T07:29:26,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:29:26,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:26,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:26,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:26,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:26,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T07:29:27,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:27,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310227096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:27,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:27,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310227096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:27,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:27,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:29:27,116 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:27,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310227096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,118 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:27,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:27,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:27,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:27,119 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:27,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:27,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:27,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310227125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,138 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:27,151 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:27,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310227139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,155 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:27,268 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/4dc0978f596d4cdfbc97943f653b6cfe 2024-11-11T07:29:27,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:29:27,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:27,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:27,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:27,275 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:27,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:27,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:27,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/6ddf9490083b493f91ac6c7e7f92e8b6 is 50, key is test_row_0/C:col10/1731310166191/Put/seqid=0 2024-11-11T07:29:27,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741860_1036 (size=12001) 2024-11-11T07:29:27,340 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/6ddf9490083b493f91ac6c7e7f92e8b6 2024-11-11T07:29:27,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/2630135164e74bb19d74239b7bfffc34 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/2630135164e74bb19d74239b7bfffc34 2024-11-11T07:29:27,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/2630135164e74bb19d74239b7bfffc34, entries=150, sequenceid=119, filesize=11.7 K 2024-11-11T07:29:27,423 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/4dc0978f596d4cdfbc97943f653b6cfe as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/4dc0978f596d4cdfbc97943f653b6cfe 2024-11-11T07:29:27,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:29:27,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:27,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:27,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:27,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:27,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:27,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:27,477 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/4dc0978f596d4cdfbc97943f653b6cfe, entries=150, sequenceid=119, filesize=11.7 K 2024-11-11T07:29:27,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/6ddf9490083b493f91ac6c7e7f92e8b6 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6ddf9490083b493f91ac6c7e7f92e8b6 2024-11-11T07:29:27,529 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6ddf9490083b493f91ac6c7e7f92e8b6, entries=150, sequenceid=119, filesize=11.7 K 2024-11-11T07:29:27,531 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~58.95 KB/60368, heapSize ~165.95 KB/169936, currentSize=124.84 KB/127835 for 6756dec3886a521f131ee738c3465dc1 in 1327ms, sequenceid=119, compaction requested=false 2024-11-11T07:29:27,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:27,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-11T07:29:27,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:27,592 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=124.84 KB heapSize=362.48 KB 2024-11-11T07:29:27,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:27,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:27,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:27,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:27,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:27,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:27,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/581701ef4029468d8fb3079cd1c32c61 is 50, key is test_row_0/A:col10/1731310166395/Put/seqid=0 2024-11-11T07:29:27,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:27,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:27,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741861_1037 (size=12151) 2024-11-11T07:29:27,658 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/581701ef4029468d8fb3079cd1c32c61 2024-11-11T07:29:27,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:27,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310227651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:27,684 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:27,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310227667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,685 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:27,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:27,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310227678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:27,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:27,697 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310227681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,698 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:27,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:27,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310227684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:27,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:27,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/833824de1a074a7b9bb40f47acda6f41 is 50, key is test_row_0/B:col10/1731310166395/Put/seqid=0 2024-11-11T07:29:27,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741862_1038 (size=12151) 2024-11-11T07:29:27,751 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=30.91 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/833824de1a074a7b9bb40f47acda6f41 2024-11-11T07:29:27,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/3d00ff896395484bb2f0557b631947b1 is 50, key is test_row_0/C:col10/1731310166395/Put/seqid=0 2024-11-11T07:29:27,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:27,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310227795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:27,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:27,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310227807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:27,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310227807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:27,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:27,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:27,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310227811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:27,818 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:27,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741863_1039 (size=12151) 2024-11-11T07:29:27,841 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=62.62 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/3d00ff896395484bb2f0557b631947b1 2024-11-11T07:29:27,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/581701ef4029468d8fb3079cd1c32c61 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/581701ef4029468d8fb3079cd1c32c61 2024-11-11T07:29:27,869 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/581701ef4029468d8fb3079cd1c32c61, entries=150, sequenceid=153, filesize=11.9 K 2024-11-11T07:29:27,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/833824de1a074a7b9bb40f47acda6f41 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/833824de1a074a7b9bb40f47acda6f41 2024-11-11T07:29:27,885 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/833824de1a074a7b9bb40f47acda6f41, entries=150, sequenceid=153, filesize=11.9 K 2024-11-11T07:29:27,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/3d00ff896395484bb2f0557b631947b1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/3d00ff896395484bb2f0557b631947b1 2024-11-11T07:29:27,898 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/3d00ff896395484bb2f0557b631947b1, entries=150, sequenceid=153, filesize=11.9 K 2024-11-11T07:29:27,900 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~124.84 KB/127835, heapSize ~362.43 KB/371128, currentSize=53.67 KB/54960 for 6756dec3886a521f131ee738c3465dc1 in 308ms, sequenceid=153, compaction requested=true 2024-11-11T07:29:27,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:27,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:27,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-11T07:29:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-11T07:29:27,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T07:29:27,906 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9410 sec 2024-11-11T07:29:27,910 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.9610 sec 2024-11-11T07:29:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:28,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T07:29:28,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:28,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:28,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:28,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:28,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:28,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:28,041 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/57e2f6d5db3c4c43b989f3104736be97 is 50, key is test_row_0/A:col10/1731310167681/Put/seqid=0 2024-11-11T07:29:28,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741864_1040 (size=14541) 2024-11-11T07:29:28,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/57e2f6d5db3c4c43b989f3104736be97 2024-11-11T07:29:28,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-11T07:29:28,115 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:28,115 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:29:28,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-11T07:29:28,120 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:29:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T07:29:28,123 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:29:28,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:29:28,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/ae716c1fca3647da8e83189ed6167c22 is 50, key is test_row_0/B:col10/1731310167681/Put/seqid=0 2024-11-11T07:29:28,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310228111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310228131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,153 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,153 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310228136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,154 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310228139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,158 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741865_1041 (size=12151) 2024-11-11T07:29:28,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/ae716c1fca3647da8e83189ed6167c22 2024-11-11T07:29:28,204 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/09a43f92441f4d7583566a2be0e00d64 is 50, key is test_row_0/C:col10/1731310167681/Put/seqid=0 2024-11-11T07:29:28,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T07:29:28,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741866_1042 (size=12151) 2024-11-11T07:29:28,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/09a43f92441f4d7583566a2be0e00d64 2024-11-11T07:29:28,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310228245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,276 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/57e2f6d5db3c4c43b989f3104736be97 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/57e2f6d5db3c4c43b989f3104736be97 2024-11-11T07:29:28,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T07:29:28,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:28,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:28,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:28,279 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:28,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:28,284 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310228266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310228266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/57e2f6d5db3c4c43b989f3104736be97, entries=200, sequenceid=165, filesize=14.2 K 2024-11-11T07:29:28,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310228267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/ae716c1fca3647da8e83189ed6167c22 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ae716c1fca3647da8e83189ed6167c22 2024-11-11T07:29:28,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ae716c1fca3647da8e83189ed6167c22, entries=150, sequenceid=165, filesize=11.9 K 2024-11-11T07:29:28,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/09a43f92441f4d7583566a2be0e00d64 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/09a43f92441f4d7583566a2be0e00d64 2024-11-11T07:29:28,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/09a43f92441f4d7583566a2be0e00d64, entries=150, sequenceid=165, filesize=11.9 K 2024-11-11T07:29:28,340 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6756dec3886a521f131ee738c3465dc1 in 315ms, sequenceid=165, compaction requested=true 2024-11-11T07:29:28,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:28,340 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:29:28,342 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:29:28,343 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50900 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:29:28,343 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/A is initiating minor compaction (all files) 2024-11-11T07:29:28,343 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/A in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:28,344 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/e01afd75d7bd4433a72186b489374df7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/2630135164e74bb19d74239b7bfffc34, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/581701ef4029468d8fb3079cd1c32c61, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/57e2f6d5db3c4c43b989f3104736be97] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=49.7 K 2024-11-11T07:29:28,345 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting e01afd75d7bd4433a72186b489374df7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731310164194 2024-11-11T07:29:28,345 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2630135164e74bb19d74239b7bfffc34, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731310165516 2024-11-11T07:29:28,346 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 581701ef4029468d8fb3079cd1c32c61, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731310166393 2024-11-11T07:29:28,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:28,347 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:29:28,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:29:28,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:28,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:29:28,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:28,348 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57e2f6d5db3c4c43b989f3104736be97, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731310167675 2024-11-11T07:29:28,351 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:29:28,351 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/B is initiating minor compaction (all files) 2024-11-11T07:29:28,352 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/B in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:28,352 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/2501523d99f9496b8d7b32b9c6893e65, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/4dc0978f596d4cdfbc97943f653b6cfe, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/833824de1a074a7b9bb40f47acda6f41, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ae716c1fca3647da8e83189ed6167c22] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=47.4 K 2024-11-11T07:29:28,353 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 2501523d99f9496b8d7b32b9c6893e65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731310164194 2024-11-11T07:29:28,355 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 4dc0978f596d4cdfbc97943f653b6cfe, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731310165516 2024-11-11T07:29:28,356 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 833824de1a074a7b9bb40f47acda6f41, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731310166393 2024-11-11T07:29:28,358 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting ae716c1fca3647da8e83189ed6167c22, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731310167675 2024-11-11T07:29:28,383 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#A#compaction#30 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:28,384 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/d557d71835e54db19cdced17395dda7a is 50, key is test_row_0/A:col10/1731310167681/Put/seqid=0 2024-11-11T07:29:28,392 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#B#compaction#31 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:28,392 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/02a6e22c648f41ed9a1b5f18faa62abf is 50, key is test_row_0/B:col10/1731310167681/Put/seqid=0 2024-11-11T07:29:28,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-11T07:29:28,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:28,437 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:29:28,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:28,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:28,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:28,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:28,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:28,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:28,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T07:29:28,449 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741867_1043 (size=12493) 2024-11-11T07:29:28,461 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/d557d71835e54db19cdced17395dda7a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/d557d71835e54db19cdced17395dda7a 2024-11-11T07:29:28,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741868_1044 (size=12493) 2024-11-11T07:29:28,481 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/A of 6756dec3886a521f131ee738c3465dc1 into d557d71835e54db19cdced17395dda7a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:28,481 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:28,481 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/A, priority=12, startTime=1731310168340; duration=0sec 2024-11-11T07:29:28,481 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:28,481 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:A 2024-11-11T07:29:28,482 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:29:28,485 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:29:28,485 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/C is initiating minor compaction (all files) 2024-11-11T07:29:28,485 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/C in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:28,485 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/665866aabf8243fa8d7223919a804c31, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6ddf9490083b493f91ac6c7e7f92e8b6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/3d00ff896395484bb2f0557b631947b1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/09a43f92441f4d7583566a2be0e00d64] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=47.4 K 2024-11-11T07:29:28,486 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 665866aabf8243fa8d7223919a804c31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731310164194 2024-11-11T07:29:28,488 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ddf9490083b493f91ac6c7e7f92e8b6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731310165516 2024-11-11T07:29:28,488 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d00ff896395484bb2f0557b631947b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731310166393 2024-11-11T07:29:28,489 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09a43f92441f4d7583566a2be0e00d64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731310167675 2024-11-11T07:29:28,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/f4794c295d7f4e40a9dbe4258a117ac1 is 50, key is test_row_0/A:col10/1731310168068/Put/seqid=0 2024-11-11T07:29:28,504 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/02a6e22c648f41ed9a1b5f18faa62abf as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/02a6e22c648f41ed9a1b5f18faa62abf 2024-11-11T07:29:28,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:28,519 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/B of 6756dec3886a521f131ee738c3465dc1 into 02a6e22c648f41ed9a1b5f18faa62abf(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:28,519 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:28,520 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/B, priority=12, startTime=1731310168347; duration=0sec 2024-11-11T07:29:28,520 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:28,520 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:B 2024-11-11T07:29:28,546 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:28,546 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/c891743d72dd44179db7f74b444550c0 is 50, key is test_row_0/C:col10/1731310167681/Put/seqid=0 2024-11-11T07:29:28,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310228539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,558 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310228542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310228545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310228549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741869_1045 (size=12151) 2024-11-11T07:29:28,585 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/f4794c295d7f4e40a9dbe4258a117ac1 2024-11-11T07:29:28,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741870_1046 (size=12493) 2024-11-11T07:29:28,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/df8d487b46844c39a05489f2994b941a is 50, key is test_row_0/B:col10/1731310168068/Put/seqid=0 2024-11-11T07:29:28,629 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/c891743d72dd44179db7f74b444550c0 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/c891743d72dd44179db7f74b444550c0 2024-11-11T07:29:28,641 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/C of 6756dec3886a521f131ee738c3465dc1 into c891743d72dd44179db7f74b444550c0(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:28,641 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:28,641 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/C, priority=12, startTime=1731310168347; duration=0sec 2024-11-11T07:29:28,641 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:28,641 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:C 2024-11-11T07:29:28,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741871_1047 (size=12151) 2024-11-11T07:29:28,710 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/df8d487b46844c39a05489f2994b941a 2024-11-11T07:29:28,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310228699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310228700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310228701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,734 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310228701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,735 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310228705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:28,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/97295379671d4834b809216f4d6f9958 is 50, key is test_row_0/C:col10/1731310168068/Put/seqid=0 2024-11-11T07:29:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,739 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_2' in 6756dec3886a521f131ee738c3465dc1 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2322 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:29:28,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:28,741 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:28,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T07:29:28,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741872_1048 (size=12151) 2024-11-11T07:29:28,816 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=185 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/97295379671d4834b809216f4d6f9958 2024-11-11T07:29:28,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/f4794c295d7f4e40a9dbe4258a117ac1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/f4794c295d7f4e40a9dbe4258a117ac1 2024-11-11T07:29:28,839 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/f4794c295d7f4e40a9dbe4258a117ac1, entries=150, sequenceid=185, filesize=11.9 K 2024-11-11T07:29:28,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/df8d487b46844c39a05489f2994b941a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/df8d487b46844c39a05489f2994b941a 2024-11-11T07:29:28,854 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/df8d487b46844c39a05489f2994b941a, entries=150, sequenceid=185, filesize=11.9 K 2024-11-11T07:29:28,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/97295379671d4834b809216f4d6f9958 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/97295379671d4834b809216f4d6f9958 2024-11-11T07:29:28,892 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/97295379671d4834b809216f4d6f9958, entries=150, sequenceid=185, filesize=11.9 K 2024-11-11T07:29:28,911 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6756dec3886a521f131ee738c3465dc1 in 473ms, sequenceid=185, compaction requested=false 2024-11-11T07:29:28,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:28,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:28,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-11T07:29:28,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-11T07:29:28,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-11T07:29:28,927 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 793 msec 2024-11-11T07:29:28,931 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 812 msec 2024-11-11T07:29:28,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:28,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T07:29:28,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:28,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:28,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:28,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:28,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:28,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:28,982 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/7632bdb3c1fe418fbbca5abb6ef753b4 is 50, key is test_row_0/A:col10/1731310168943/Put/seqid=0 2024-11-11T07:29:29,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741873_1049 (size=12151) 2024-11-11T07:29:29,047 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310229032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,048 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310229032, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,054 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,055 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310229048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310229037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310229155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,157 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,175 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,176 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310229165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,179 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310229165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310229195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,205 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-11T07:29:29,255 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:29,260 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:29:29,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-11T07:29:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T07:29:29,270 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:29:29,272 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:29:29,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:29:29,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310229365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,371 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T07:29:29,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310229384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,391 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310229384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,395 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,417 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310229415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,418 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,419 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,423 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/7632bdb3c1fe418fbbca5abb6ef753b4 2024-11-11T07:29:29,431 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T07:29:29,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:29,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:29,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:29,432 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:29,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:29,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:29,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/72a85c31adc84d298125bd9c50e710a0 is 50, key is test_row_0/B:col10/1731310168943/Put/seqid=0 2024-11-11T07:29:29,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741874_1050 (size=12151) 2024-11-11T07:29:29,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/72a85c31adc84d298125bd9c50e710a0 2024-11-11T07:29:29,534 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/a0c1a08c5f1046f6812adfcc6ea3c633 is 50, key is test_row_0/C:col10/1731310168943/Put/seqid=0 2024-11-11T07:29:29,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T07:29:29,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741875_1051 (size=12151) 2024-11-11T07:29:29,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=202 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/a0c1a08c5f1046f6812adfcc6ea3c633 2024-11-11T07:29:29,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T07:29:29,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:29,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:29,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:29,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:29,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:29,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:29,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/7632bdb3c1fe418fbbca5abb6ef753b4 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/7632bdb3c1fe418fbbca5abb6ef753b4 2024-11-11T07:29:29,618 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/7632bdb3c1fe418fbbca5abb6ef753b4, entries=150, sequenceid=202, filesize=11.9 K 2024-11-11T07:29:29,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/72a85c31adc84d298125bd9c50e710a0 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/72a85c31adc84d298125bd9c50e710a0 2024-11-11T07:29:29,644 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/72a85c31adc84d298125bd9c50e710a0, entries=150, sequenceid=202, filesize=11.9 K 2024-11-11T07:29:29,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/a0c1a08c5f1046f6812adfcc6ea3c633 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/a0c1a08c5f1046f6812adfcc6ea3c633 2024-11-11T07:29:29,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/a0c1a08c5f1046f6812adfcc6ea3c633, entries=150, sequenceid=202, filesize=11.9 K 2024-11-11T07:29:29,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6756dec3886a521f131ee738c3465dc1 in 704ms, sequenceid=202, compaction requested=true 2024-11-11T07:29:29,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:29,662 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:29,664 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:29,664 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/A is initiating minor compaction (all files) 2024-11-11T07:29:29,665 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/A in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:29,665 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/d557d71835e54db19cdced17395dda7a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/f4794c295d7f4e40a9dbe4258a117ac1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/7632bdb3c1fe418fbbca5abb6ef753b4] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=35.9 K 2024-11-11T07:29:29,666 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting d557d71835e54db19cdced17395dda7a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731310167675 2024-11-11T07:29:29,666 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4794c295d7f4e40a9dbe4258a117ac1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731310168068 2024-11-11T07:29:29,667 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7632bdb3c1fe418fbbca5abb6ef753b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731310168540 2024-11-11T07:29:29,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:29:29,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:29,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:29:29,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:29,678 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:29,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:29:29,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:29,680 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:29,680 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/B is initiating minor compaction (all files) 2024-11-11T07:29:29,681 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/B in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:29,681 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/02a6e22c648f41ed9a1b5f18faa62abf, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/df8d487b46844c39a05489f2994b941a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/72a85c31adc84d298125bd9c50e710a0] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=35.9 K 2024-11-11T07:29:29,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:29,682 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 02a6e22c648f41ed9a1b5f18faa62abf, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731310167675 2024-11-11T07:29:29,682 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting df8d487b46844c39a05489f2994b941a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731310168068 2024-11-11T07:29:29,683 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 72a85c31adc84d298125bd9c50e710a0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731310168540 2024-11-11T07:29:29,696 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#A#compaction#39 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:29,697 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/316011ad1d8c43f38fd390b7976d9b34 is 50, key is test_row_0/A:col10/1731310168943/Put/seqid=0 2024-11-11T07:29:29,713 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:29:29,713 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#B#compaction#40 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:29,714 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/68f01577b36e484db801284b81e11699 is 50, key is test_row_0/B:col10/1731310168943/Put/seqid=0 2024-11-11T07:29:29,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:29,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:29,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:29,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:29,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:29,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:29,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741876_1052 (size=12595) 2024-11-11T07:29:29,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/26def2790f794d4694191a84c32aa3e3 is 50, key is test_row_0/A:col10/1731310169709/Put/seqid=0 2024-11-11T07:29:29,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741877_1053 (size=12595) 2024-11-11T07:29:29,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T07:29:29,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:29,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:29,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:29,756 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:29,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:29,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:29,761 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/316011ad1d8c43f38fd390b7976d9b34 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/316011ad1d8c43f38fd390b7976d9b34 2024-11-11T07:29:29,765 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/68f01577b36e484db801284b81e11699 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/68f01577b36e484db801284b81e11699 2024-11-11T07:29:29,796 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/B of 6756dec3886a521f131ee738c3465dc1 into 68f01577b36e484db801284b81e11699(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:29,796 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:29,796 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/B, priority=13, startTime=1731310169678; duration=0sec 2024-11-11T07:29:29,796 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:29,796 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:B 2024-11-11T07:29:29,797 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:29,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741878_1054 (size=12151) 2024-11-11T07:29:29,798 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/A of 6756dec3886a521f131ee738c3465dc1 into 316011ad1d8c43f38fd390b7976d9b34(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:29,798 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:29,798 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/A, priority=13, startTime=1731310169662; duration=0sec 2024-11-11T07:29:29,799 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:29,799 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:A 2024-11-11T07:29:29,800 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/26def2790f794d4694191a84c32aa3e3 2024-11-11T07:29:29,800 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:29,801 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/C is initiating minor compaction (all files) 2024-11-11T07:29:29,802 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/C in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:29,802 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/c891743d72dd44179db7f74b444550c0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/97295379671d4834b809216f4d6f9958, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/a0c1a08c5f1046f6812adfcc6ea3c633] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=35.9 K 2024-11-11T07:29:29,804 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting c891743d72dd44179db7f74b444550c0, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731310167675 2024-11-11T07:29:29,805 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 97295379671d4834b809216f4d6f9958, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=185, earliestPutTs=1731310168068 2024-11-11T07:29:29,805 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting a0c1a08c5f1046f6812adfcc6ea3c633, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731310168540 2024-11-11T07:29:29,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/a91ea190016d486bbf94283104166420 is 50, key is test_row_0/B:col10/1731310169709/Put/seqid=0 2024-11-11T07:29:29,838 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#C#compaction#42 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:29,839 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/df102dc92683480dae118dacad15df46 is 50, key is test_row_0/C:col10/1731310168943/Put/seqid=0 2024-11-11T07:29:29,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310229834, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,867 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310229838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310229848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,875 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310229849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,876 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741880_1056 (size=12151) 2024-11-11T07:29:29,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T07:29:29,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/a91ea190016d486bbf94283104166420 2024-11-11T07:29:29,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T07:29:29,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:29,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:29,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:29,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:29,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:29,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:29,929 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/1c9823913b1b4021a2f1a294bbf1136d is 50, key is test_row_0/C:col10/1731310169709/Put/seqid=0 2024-11-11T07:29:29,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741879_1055 (size=12595) 2024-11-11T07:29:29,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741881_1057 (size=12151) 2024-11-11T07:29:29,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/1c9823913b1b4021a2f1a294bbf1136d 2024-11-11T07:29:29,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310229977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:29,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310229977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:29,986 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:29,997 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/26def2790f794d4694191a84c32aa3e3 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/26def2790f794d4694191a84c32aa3e3 2024-11-11T07:29:30,004 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/26def2790f794d4694191a84c32aa3e3, entries=150, sequenceid=223, filesize=11.9 K 2024-11-11T07:29:30,005 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/a91ea190016d486bbf94283104166420 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/a91ea190016d486bbf94283104166420 2024-11-11T07:29:30,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:30,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310229988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:30,010 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:30,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310229991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,012 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:30,014 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/a91ea190016d486bbf94283104166420, entries=150, sequenceid=223, filesize=11.9 K 2024-11-11T07:29:30,016 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/1c9823913b1b4021a2f1a294bbf1136d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/1c9823913b1b4021a2f1a294bbf1136d 2024-11-11T07:29:30,028 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/1c9823913b1b4021a2f1a294bbf1136d, entries=150, sequenceid=223, filesize=11.9 K 2024-11-11T07:29:30,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 6756dec3886a521f131ee738c3465dc1 in 317ms, sequenceid=223, compaction requested=false 2024-11-11T07:29:30,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:30,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-11T07:29:30,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:30,074 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T07:29:30,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:30,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:30,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:30,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:30,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:30,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:30,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/0399fd0f411b4d0bbda6ae7532947c90 is 50, key is test_row_0/A:col10/1731310169827/Put/seqid=0 2024-11-11T07:29:30,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741882_1058 (size=12151) 2024-11-11T07:29:30,133 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/0399fd0f411b4d0bbda6ae7532947c90 2024-11-11T07:29:30,156 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/d234de99693144ebb32bbdf269c5c174 is 50, key is test_row_0/B:col10/1731310169827/Put/seqid=0 2024-11-11T07:29:30,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741883_1059 (size=12151) 2024-11-11T07:29:30,200 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/d234de99693144ebb32bbdf269c5c174 2024-11-11T07:29:30,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:30,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:30,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/46f2faec2361461b84a08987806b0091 is 50, key is test_row_0/C:col10/1731310169827/Put/seqid=0 2024-11-11T07:29:30,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741884_1060 (size=12151) 2024-11-11T07:29:30,287 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=236 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/46f2faec2361461b84a08987806b0091 2024-11-11T07:29:30,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/0399fd0f411b4d0bbda6ae7532947c90 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/0399fd0f411b4d0bbda6ae7532947c90 2024-11-11T07:29:30,312 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/0399fd0f411b4d0bbda6ae7532947c90, entries=150, sequenceid=236, filesize=11.9 K 2024-11-11T07:29:30,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/d234de99693144ebb32bbdf269c5c174 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/d234de99693144ebb32bbdf269c5c174 2024-11-11T07:29:30,332 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/d234de99693144ebb32bbdf269c5c174, entries=150, sequenceid=236, filesize=11.9 K 2024-11-11T07:29:30,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/46f2faec2361461b84a08987806b0091 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/46f2faec2361461b84a08987806b0091 2024-11-11T07:29:30,351 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/df102dc92683480dae118dacad15df46 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/df102dc92683480dae118dacad15df46 2024-11-11T07:29:30,359 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/46f2faec2361461b84a08987806b0091, entries=150, sequenceid=236, filesize=11.9 K 2024-11-11T07:29:30,362 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=80.51 KB/82440 for 6756dec3886a521f131ee738c3465dc1 in 287ms, sequenceid=236, compaction requested=true 2024-11-11T07:29:30,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:30,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:30,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-11T07:29:30,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-11T07:29:30,367 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/C of 6756dec3886a521f131ee738c3465dc1 into df102dc92683480dae118dacad15df46(size=12.3 K), total size for store is 36.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:30,367 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:30,367 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/C, priority=13, startTime=1731310169678; duration=0sec 2024-11-11T07:29:30,367 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:30,367 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:C 2024-11-11T07:29:30,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-11T07:29:30,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0930 sec 2024-11-11T07:29:30,372 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 1.1100 sec 2024-11-11T07:29:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:30,383 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T07:29:30,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:30,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:30,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:30,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:30,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:30,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:30,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-11T07:29:30,405 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:30,406 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:29:30,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/fe69164e57c64fb2a5a6c0c2c2f130d1 is 50, key is test_row_0/A:col10/1731310170267/Put/seqid=0 2024-11-11T07:29:30,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-11T07:29:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T07:29:30,413 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:29:30,415 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:29:30,415 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:29:30,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741885_1061 (size=21715) 2024-11-11T07:29:30,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/fe69164e57c64fb2a5a6c0c2c2f130d1 2024-11-11T07:29:30,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/f62ba4ff0d37483a95ff0560ff5aeee2 is 50, key is test_row_0/B:col10/1731310170267/Put/seqid=0 2024-11-11T07:29:30,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310230468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:30,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,490 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:30,494 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:30,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310230480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:30,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741886_1062 (size=12151) 2024-11-11T07:29:30,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,496 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:30,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:30,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/f62ba4ff0d37483a95ff0560ff5aeee2 2024-11-11T07:29:30,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310230487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:30,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,499 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:30,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310230489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:30,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,503 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T07:29:30,519 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/89e8a61365a74b108a2dbb0dde1e9431 is 50, key is test_row_0/C:col10/1731310170267/Put/seqid=0 2024-11-11T07:29:30,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741887_1063 (size=12151) 2024-11-11T07:29:30,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T07:29:30,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:30,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:30,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:30,571 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:30,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:30,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:30,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:30,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310230595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,609 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:30,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310230606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:30,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:30,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:30,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310230606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310230607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,622 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:30,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T07:29:30,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T07:29:30,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:30,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:30,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:30,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:30,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:30,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:30,756 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310230755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:30,757 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_2' in 6756dec3886a521f131ee738c3465dc1 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4343 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:29:30,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:30,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:30,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310230815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:30,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,825 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:30,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310230825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,829 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:30,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:30,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310230825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:30,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,830 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,831 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:30,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:30,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310230827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,833 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:30,834 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:30,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T07:29:30,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:30,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:30,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:30,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:30,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:30,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:30,968 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/89e8a61365a74b108a2dbb0dde1e9431 2024-11-11T07:29:30,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/fe69164e57c64fb2a5a6c0c2c2f130d1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/fe69164e57c64fb2a5a6c0c2c2f130d1 2024-11-11T07:29:30,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/fe69164e57c64fb2a5a6c0c2c2f130d1, entries=350, sequenceid=256, filesize=21.2 K 2024-11-11T07:29:30,988 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/f62ba4ff0d37483a95ff0560ff5aeee2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/f62ba4ff0d37483a95ff0560ff5aeee2 2024-11-11T07:29:31,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/f62ba4ff0d37483a95ff0560ff5aeee2, entries=150, sequenceid=256, filesize=11.9 K 2024-11-11T07:29:31,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/89e8a61365a74b108a2dbb0dde1e9431 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/89e8a61365a74b108a2dbb0dde1e9431 2024-11-11T07:29:31,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/89e8a61365a74b108a2dbb0dde1e9431, entries=150, sequenceid=256, filesize=11.9 K 2024-11-11T07:29:31,021 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 6756dec3886a521f131ee738c3465dc1 in 639ms, sequenceid=256, compaction requested=true 2024-11-11T07:29:31,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:31,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:29:31,021 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:29:31,021 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:31,021 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:29:31,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:29:31,026 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 58612 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:29:31,026 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/A is initiating minor compaction (all files) 2024-11-11T07:29:31,026 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/A in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:31,026 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:29:31,026 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/B is initiating minor compaction (all files) 2024-11-11T07:29:31,027 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/B in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:31,027 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/68f01577b36e484db801284b81e11699, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/a91ea190016d486bbf94283104166420, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/d234de99693144ebb32bbdf269c5c174, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/f62ba4ff0d37483a95ff0560ff5aeee2] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=47.9 K 2024-11-11T07:29:31,027 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/316011ad1d8c43f38fd390b7976d9b34, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/26def2790f794d4694191a84c32aa3e3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/0399fd0f411b4d0bbda6ae7532947c90, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/fe69164e57c64fb2a5a6c0c2c2f130d1] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=57.2 K 2024-11-11T07:29:31,028 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 316011ad1d8c43f38fd390b7976d9b34, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731310168540 2024-11-11T07:29:31,028 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 68f01577b36e484db801284b81e11699, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731310168540 2024-11-11T07:29:31,029 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26def2790f794d4694191a84c32aa3e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731310169030 2024-11-11T07:29:31,029 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting a91ea190016d486bbf94283104166420, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731310169030 2024-11-11T07:29:31,030 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0399fd0f411b4d0bbda6ae7532947c90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731310169827 2024-11-11T07:29:31,030 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting d234de99693144ebb32bbdf269c5c174, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731310169827 2024-11-11T07:29:31,030 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting fe69164e57c64fb2a5a6c0c2c2f130d1, keycount=350, bloomtype=ROW, size=21.2 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731310170237 2024-11-11T07:29:31,031 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f62ba4ff0d37483a95ff0560ff5aeee2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731310170267 2024-11-11T07:29:31,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T07:29:31,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-11T07:29:31,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:31,042 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T07:29:31,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:31,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:31,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:31,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:31,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:31,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:31,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/c812fd10ce444f6bac0264ec2029eb4a is 50, key is test_row_0/A:col10/1731310170486/Put/seqid=0 2024-11-11T07:29:31,057 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:31,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:29:31,058 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:31,077 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#B#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:31,078 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/f3de1cc47000450b9b93bb5898e1682b is 50, key is test_row_0/B:col10/1731310170267/Put/seqid=0 2024-11-11T07:29:31,082 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#A#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:31,082 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/9b525c81b620403aab11dbd302791120 is 50, key is test_row_0/A:col10/1731310170267/Put/seqid=0 2024-11-11T07:29:31,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741888_1064 (size=12301) 2024-11-11T07:29:31,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741889_1065 (size=12731) 2024-11-11T07:29:31,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:31,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:31,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741890_1066 (size=12731) 2024-11-11T07:29:31,174 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/9b525c81b620403aab11dbd302791120 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/9b525c81b620403aab11dbd302791120 2024-11-11T07:29:31,183 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/A of 6756dec3886a521f131ee738c3465dc1 into 9b525c81b620403aab11dbd302791120(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:31,183 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:31,184 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/A, priority=12, startTime=1731310171021; duration=0sec 2024-11-11T07:29:31,184 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:31,184 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:A 2024-11-11T07:29:31,184 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:29:31,186 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:29:31,186 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/C is initiating minor compaction (all files) 2024-11-11T07:29:31,186 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/C in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:31,186 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/df102dc92683480dae118dacad15df46, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/1c9823913b1b4021a2f1a294bbf1136d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/46f2faec2361461b84a08987806b0091, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/89e8a61365a74b108a2dbb0dde1e9431] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=47.9 K 2024-11-11T07:29:31,187 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting df102dc92683480dae118dacad15df46, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=202, earliestPutTs=1731310168540 2024-11-11T07:29:31,187 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c9823913b1b4021a2f1a294bbf1136d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731310169030 2024-11-11T07:29:31,188 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 46f2faec2361461b84a08987806b0091, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=236, earliestPutTs=1731310169827 2024-11-11T07:29:31,188 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89e8a61365a74b108a2dbb0dde1e9431, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731310170267 2024-11-11T07:29:31,211 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#C#compaction#54 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:31,211 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/b790366eebf747a0b3c4c01f403e0d59 is 50, key is test_row_0/C:col10/1731310170267/Put/seqid=0 2024-11-11T07:29:31,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:31,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310231220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:31,234 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:31,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310231223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:31,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,235 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:31,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741891_1067 (size=12731) 2024-11-11T07:29:31,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310231233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:31,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:31,250 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310231233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:31,262 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/b790366eebf747a0b3c4c01f403e0d59 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/b790366eebf747a0b3c4c01f403e0d59 2024-11-11T07:29:31,273 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/C of 6756dec3886a521f131ee738c3465dc1 into b790366eebf747a0b3c4c01f403e0d59(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:31,273 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:31,274 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/C, priority=12, startTime=1731310171058; duration=0sec 2024-11-11T07:29:31,274 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:31,274 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:C 2024-11-11T07:29:31,356 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310231345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:31,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:31,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310231345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,358 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:31,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:31,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310231355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,365 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:31,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:31,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310231357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:31,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:31,545 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/c812fd10ce444f6bac0264ec2029eb4a 2024-11-11T07:29:31,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T07:29:31,566 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/f3de1cc47000450b9b93bb5898e1682b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/f3de1cc47000450b9b93bb5898e1682b 2024-11-11T07:29:31,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:31,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310231567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:31,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:31,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/34155f8b9b14479e97c0a64c537d08ed is 50, key is test_row_0/B:col10/1731310170486/Put/seqid=0 2024-11-11T07:29:31,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:31,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310231579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:31,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310231580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:31,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:31,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:31,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310231580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,589 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,590 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:31,590 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/B of 6756dec3886a521f131ee738c3465dc1 into f3de1cc47000450b9b93bb5898e1682b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:31,590 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:31,590 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/B, priority=12, startTime=1731310171021; duration=0sec 2024-11-11T07:29:31,591 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:31,591 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:B 2024-11-11T07:29:31,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741892_1068 (size=12301) 2024-11-11T07:29:31,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:31,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310231885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,887 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:31,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:31,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310231895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:31,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:31,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310231895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:31,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:31,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310231895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:31,901 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:32,027 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/34155f8b9b14479e97c0a64c537d08ed 2024-11-11T07:29:32,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/c81f07e158214bf981190161583947a7 is 50, key is test_row_0/C:col10/1731310170486/Put/seqid=0 2024-11-11T07:29:32,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741893_1069 (size=12301) 2024-11-11T07:29:32,096 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/c81f07e158214bf981190161583947a7 2024-11-11T07:29:32,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/c812fd10ce444f6bac0264ec2029eb4a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/c812fd10ce444f6bac0264ec2029eb4a 2024-11-11T07:29:32,137 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/c812fd10ce444f6bac0264ec2029eb4a, entries=150, sequenceid=270, filesize=12.0 K 2024-11-11T07:29:32,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/34155f8b9b14479e97c0a64c537d08ed as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/34155f8b9b14479e97c0a64c537d08ed 2024-11-11T07:29:32,160 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/34155f8b9b14479e97c0a64c537d08ed, entries=150, sequenceid=270, filesize=12.0 K 2024-11-11T07:29:32,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/c81f07e158214bf981190161583947a7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/c81f07e158214bf981190161583947a7 2024-11-11T07:29:32,170 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/c81f07e158214bf981190161583947a7, entries=150, sequenceid=270, filesize=12.0 K 2024-11-11T07:29:32,172 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 6756dec3886a521f131ee738c3465dc1 in 1129ms, sequenceid=270, compaction requested=false 2024-11-11T07:29:32,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:32,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:32,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-11T07:29:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-11T07:29:32,178 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-11T07:29:32,179 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7590 sec 2024-11-11T07:29:32,182 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.7740 sec 2024-11-11T07:29:32,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:32,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:29:32,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310171211, max timestamp=1731310172410Num uniques 859; 2024-11-11T07:29:32,417 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:32,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310171211, max timestamp=1731310172413Num uniques 859; 2024-11-11T07:29:32,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310171211, max timestamp=1731310172413Num uniques 859; 2024-11-11T07:29:32,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:32,418 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:32,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:32,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:32,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:32,419 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:32,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:32,420 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:32,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/371429fd1da64bc0b2507cd703f215ce is 50, key is test_row_0/A:col10/1731310172410/Put/seqid=0 2024-11-11T07:29:32,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741894_1070 (size=17181) 2024-11-11T07:29:32,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731310172497, max timestamp=1731310172531Num uniques 859; 2024-11-11T07:29:32,535 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T07:29:32,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310172497, max timestamp=1731310172531Num uniques 859; 2024-11-11T07:29:32,541 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:32,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310172497, max timestamp=1731310172532Num uniques 859; 2024-11-11T07:29:32,542 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:32,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-11T07:29:32,566 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:32,567 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:29:32,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-11T07:29:32,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T07:29:32,572 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:29:32,574 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:29:32,574 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:29:32,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T07:29:32,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310232692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:32,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:32,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310232697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:32,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731310172599, max timestamp=1731310172666Num uniques 1068; 2024-11-11T07:29:32,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48900, new segment data size=6870, suffix heap size=139040,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=150 2024-11-11T07:29:32,713 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731310172599, max timestamp=1731310172666Num uniques 1018; 2024-11-11T07:29:32,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:32,714 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-11T07:29:32,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:32,714 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:32,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:32,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:32,715 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:32,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731310172599, max timestamp=1731310172666Num uniques 1018; 2024-11-11T07:29:32,717 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46610, new segment data size=6870, suffix heap size=133840,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=150 2024-11-11T07:29:32,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:29:32,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:32,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310232684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:32,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:32,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:32,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:32,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:32,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:32,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:32,735 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:32,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:32,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/371429fd1da64bc0b2507cd703f215ce 2024-11-11T07:29:32,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/0aadbfd67bf2456fb7ef98b55754f4c3 is 50, key is test_row_0/B:col10/1731310172410/Put/seqid=0 2024-11-11T07:29:32,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T07:29:32,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:29:32,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:32,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:32,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:32,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:32,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:32,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:32,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741895_1071 (size=12301) 2024-11-11T07:29:32,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/0aadbfd67bf2456fb7ef98b55754f4c3 2024-11-11T07:29:32,936 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/d3bcd016fcb74b31b312aae6840bbc46 is 50, key is test_row_0/C:col10/1731310172410/Put/seqid=0 2024-11-11T07:29:32,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731310172842, max timestamp=1731310172952Num uniques 1009; 2024-11-11T07:29:32,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46222, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-11T07:29:32,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731310172842, max timestamp=1731310172952Num uniques 1009; 2024-11-11T07:29:32,963 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-11T07:29:32,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731310172842, max timestamp=1731310172952Num uniques 1009; 2024-11-11T07:29:32,965 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46222, new segment data size=10392, suffix heap size=132904,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=227 2024-11-11T07:29:32,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741896_1072 (size=12301) 2024-11-11T07:29:32,981 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=295 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/d3bcd016fcb74b31b312aae6840bbc46 2024-11-11T07:29:33,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/371429fd1da64bc0b2507cd703f215ce as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/371429fd1da64bc0b2507cd703f215ce 2024-11-11T07:29:33,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/371429fd1da64bc0b2507cd703f215ce, entries=250, sequenceid=295, filesize=16.8 K 2024-11-11T07:29:33,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/0aadbfd67bf2456fb7ef98b55754f4c3 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/0aadbfd67bf2456fb7ef98b55754f4c3 2024-11-11T07:29:33,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/0aadbfd67bf2456fb7ef98b55754f4c3, entries=150, sequenceid=295, filesize=12.0 K 2024-11-11T07:29:33,034 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/d3bcd016fcb74b31b312aae6840bbc46 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/d3bcd016fcb74b31b312aae6840bbc46 2024-11-11T07:29:33,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:29:33,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:33,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,046 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,048 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/d3bcd016fcb74b31b312aae6840bbc46, entries=150, sequenceid=295, filesize=12.0 K 2024-11-11T07:29:33,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~86.11 KB/88176, currentSize=127.47 KB/130530 for 6756dec3886a521f131ee738c3465dc1 in 638ms, sequenceid=295, compaction requested=true 2024-11-11T07:29:33,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:33,052 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:33,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:29:33,052 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:33,053 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:33,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:29:33,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:33,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:29:33,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:33,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=134.18 KB heapSize=378.61 KB 2024-11-11T07:29:33,053 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:33,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:33,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:33,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:33,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:33,054 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42213 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:33,055 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:33,055 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/A is initiating minor compaction (all files) 2024-11-11T07:29:33,055 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/A in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,055 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/9b525c81b620403aab11dbd302791120, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/c812fd10ce444f6bac0264ec2029eb4a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/371429fd1da64bc0b2507cd703f215ce] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=41.2 K 2024-11-11T07:29:33,056 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:33,056 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/B is initiating minor compaction (all files) 2024-11-11T07:29:33,056 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/B in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,056 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/f3de1cc47000450b9b93bb5898e1682b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/34155f8b9b14479e97c0a64c537d08ed, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/0aadbfd67bf2456fb7ef98b55754f4c3] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=36.5 K 2024-11-11T07:29:33,056 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b525c81b620403aab11dbd302791120, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731310170267 2024-11-11T07:29:33,057 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f3de1cc47000450b9b93bb5898e1682b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731310170267 2024-11-11T07:29:33,057 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting c812fd10ce444f6bac0264ec2029eb4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731310170443 2024-11-11T07:29:33,058 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 34155f8b9b14479e97c0a64c537d08ed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731310170443 2024-11-11T07:29:33,059 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aadbfd67bf2456fb7ef98b55754f4c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731310171211 2024-11-11T07:29:33,059 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 371429fd1da64bc0b2507cd703f215ce, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731310171211 2024-11-11T07:29:33,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/45cdd05e904b4bef9212b63f81f728dd is 50, key is test_row_0/A:col10/1731310173035/Put/seqid=0 2024-11-11T07:29:33,089 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#B#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:33,089 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/763e5c4582614d399726b1aff3b9a563 is 50, key is test_row_0/B:col10/1731310172410/Put/seqid=0 2024-11-11T07:29:33,090 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:33,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310233078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,093 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:33,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310233081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,109 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:33,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:33,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310233083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:33,116 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#A#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:33,117 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/1ce997309eb84ea0b2f60e8824cfada5 is 50, key is test_row_0/A:col10/1731310172410/Put/seqid=0 2024-11-11T07:29:33,117 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,118 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:33,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:33,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310233090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,127 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:33,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741897_1073 (size=29385) 2024-11-11T07:29:33,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741898_1074 (size=12983) 2024-11-11T07:29:33,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741899_1075 (size=12983) 2024-11-11T07:29:33,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T07:29:33,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:29:33,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:33,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:33,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310233195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,209 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:33,211 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/1ce997309eb84ea0b2f60e8824cfada5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/1ce997309eb84ea0b2f60e8824cfada5 2024-11-11T07:29:33,227 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/A of 6756dec3886a521f131ee738c3465dc1 into 1ce997309eb84ea0b2f60e8824cfada5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:33,228 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:33,228 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/A, priority=13, startTime=1731310173051; duration=0sec 2024-11-11T07:29:33,228 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:33,228 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:A 2024-11-11T07:29:33,228 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:33,231 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:33,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310233214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:33,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:33,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310233226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:33,232 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:33,236 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:33,237 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/C is initiating minor compaction (all files) 2024-11-11T07:29:33,237 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/C in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,237 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/b790366eebf747a0b3c4c01f403e0d59, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/c81f07e158214bf981190161583947a7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/d3bcd016fcb74b31b312aae6840bbc46] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=36.5 K 2024-11-11T07:29:33,238 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting b790366eebf747a0b3c4c01f403e0d59, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731310170267 2024-11-11T07:29:33,238 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting c81f07e158214bf981190161583947a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731310170443 2024-11-11T07:29:33,239 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3bcd016fcb74b31b312aae6840bbc46, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731310171211 2024-11-11T07:29:33,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:33,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310233239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:33,274 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#C#compaction#63 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:33,277 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/ad1364e1c72048c79fb62185970fe194 is 50, key is test_row_0/C:col10/1731310172410/Put/seqid=0 2024-11-11T07:29:33,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741900_1076 (size=12983) 2024-11-11T07:29:33,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:29:33,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:33,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,391 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/ad1364e1c72048c79fb62185970fe194 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/ad1364e1c72048c79fb62185970fe194 2024-11-11T07:29:33,425 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:33,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310233415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,426 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:33,439 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/C of 6756dec3886a521f131ee738c3465dc1 into ad1364e1c72048c79fb62185970fe194(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:33,439 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:33,439 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/C, priority=13, startTime=1731310173053; duration=0sec 2024-11-11T07:29:33,439 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:33,439 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:C 2024-11-11T07:29:33,454 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:33,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310233435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:33,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:33,455 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:33,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310233445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:33,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,456 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:33,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:33,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310233455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:33,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:29:33,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:33,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,514 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/45cdd05e904b4bef9212b63f81f728dd 2024-11-11T07:29:33,596 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/763e5c4582614d399726b1aff3b9a563 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/763e5c4582614d399726b1aff3b9a563 2024-11-11T07:29:33,598 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/cc82b143f01c4d099bcf57b01d32ac54 is 50, key is test_row_0/B:col10/1731310173035/Put/seqid=0 2024-11-11T07:29:33,604 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/B of 6756dec3886a521f131ee738c3465dc1 into 763e5c4582614d399726b1aff3b9a563(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:33,604 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:33,604 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/B, priority=13, startTime=1731310173052; duration=0sec 2024-11-11T07:29:33,605 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:33,605 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:B 2024-11-11T07:29:33,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741901_1077 (size=12301) 2024-11-11T07:29:33,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/cc82b143f01c4d099bcf57b01d32ac54 2024-11-11T07:29:33,644 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/9dcb094dbd71427f8dfce24285e3cded is 50, key is test_row_0/C:col10/1731310173035/Put/seqid=0 2024-11-11T07:29:33,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741902_1078 (size=12301) 2024-11-11T07:29:33,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:29:33,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:33,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T07:29:33,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310233735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:33,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:33,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:33,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310233766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:33,769 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:33,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310233766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:33,769 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,770 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:33,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:33,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310233791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:33,807 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:33,808 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:33,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:29:33,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:33,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:29:33,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:33,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:33,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:33,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:34,071 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/9dcb094dbd71427f8dfce24285e3cded 2024-11-11T07:29:34,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/45cdd05e904b4bef9212b63f81f728dd as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/45cdd05e904b4bef9212b63f81f728dd 2024-11-11T07:29:34,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:29:34,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:34,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:34,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:34,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:34,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:34,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/45cdd05e904b4bef9212b63f81f728dd, entries=500, sequenceid=365, filesize=28.7 K 2024-11-11T07:29:34,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:34,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/cc82b143f01c4d099bcf57b01d32ac54 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/cc82b143f01c4d099bcf57b01d32ac54 2024-11-11T07:29:34,183 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/cc82b143f01c4d099bcf57b01d32ac54, entries=150, sequenceid=365, filesize=12.0 K 2024-11-11T07:29:34,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/9dcb094dbd71427f8dfce24285e3cded as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/9dcb094dbd71427f8dfce24285e3cded 2024-11-11T07:29:34,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/9dcb094dbd71427f8dfce24285e3cded, entries=150, sequenceid=365, filesize=12.0 K 2024-11-11T07:29:34,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~378.56 KB/387648, currentSize=46.96 KB/48090 for 6756dec3886a521f131ee738c3465dc1 in 1161ms, sequenceid=365, compaction requested=false 2024-11-11T07:29:34,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:34,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T07:29:34,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:34,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:34,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:34,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:34,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:34,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:34,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:34,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/61994602579047d4b484691b6ec67585 is 50, key is test_row_0/A:col10/1731310173078/Put/seqid=0 2024-11-11T07:29:34,309 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:29:34,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741903_1079 (size=12297) 2024-11-11T07:29:34,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:34,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:34,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:34,310 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:34,310 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:34,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/61994602579047d4b484691b6ec67585 2024-11-11T07:29:34,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:34,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/933bf28a59554c88a8afc43be9e4c16a is 50, key is test_row_0/B:col10/1731310173078/Put/seqid=0 2024-11-11T07:29:34,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741904_1080 (size=9857) 2024-11-11T07:29:34,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/933bf28a59554c88a8afc43be9e4c16a 2024-11-11T07:29:34,424 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/6fd4af622be44a8f95e386c8f56790b6 is 50, key is test_row_0/C:col10/1731310173078/Put/seqid=0 2024-11-11T07:29:34,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:29:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:34,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:34,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:34,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:34,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:34,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:34,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310234511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,517 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:34,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:34,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310234511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:34,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,519 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:34,520 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:34,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310234510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:34,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:34,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:34,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310234517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,524 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:34,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741905_1081 (size=9857) 2024-11-11T07:29:34,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310174440, max timestamp=1731310174511Num uniques 859; 2024-11-11T07:29:34,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:34,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310174440, max timestamp=1731310174511Num uniques 859; 2024-11-11T07:29:34,531 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:34,532 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310174440, max timestamp=1731310174511Num uniques 859; 2024-11-11T07:29:34,534 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:34,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:29:34,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:34,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:34,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:34,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:34,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:34,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:34,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:34,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310234677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:34,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:34,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310234684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:34,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310234684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,693 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:34,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:34,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310234686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:34,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,713 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:34,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T07:29:34,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:34,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34362 deadline: 1731310234775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:34,787 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_2' in 6756dec3886a521f131ee738c3465dc1 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8372 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:29:34,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,788 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:34,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:34,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310234795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,804 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:34,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310234795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:34,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310234804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:34,811 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,812 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:34,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:34,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310234815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:34,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:34,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:29:34,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:34,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:34,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:34,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:34,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:34,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:34,925 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=379 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/6fd4af622be44a8f95e386c8f56790b6 2024-11-11T07:29:34,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/61994602579047d4b484691b6ec67585 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/61994602579047d4b484691b6ec67585 2024-11-11T07:29:34,944 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/61994602579047d4b484691b6ec67585, entries=150, sequenceid=379, filesize=12.0 K 2024-11-11T07:29:34,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/933bf28a59554c88a8afc43be9e4c16a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/933bf28a59554c88a8afc43be9e4c16a 2024-11-11T07:29:34,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/933bf28a59554c88a8afc43be9e4c16a, entries=100, sequenceid=379, filesize=9.6 K 2024-11-11T07:29:34,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/6fd4af622be44a8f95e386c8f56790b6 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6fd4af622be44a8f95e386c8f56790b6 2024-11-11T07:29:34,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6fd4af622be44a8f95e386c8f56790b6, entries=100, sequenceid=379, filesize=9.6 K 2024-11-11T07:29:34,973 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6756dec3886a521f131ee738c3465dc1 in 712ms, sequenceid=379, compaction requested=true 2024-11-11T07:29:34,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:34,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:29:34,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:34,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:29:34,973 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:34,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:34,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:29:34,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:29:34,974 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:34,980 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 54665 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:34,980 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/A is initiating minor compaction (all files) 2024-11-11T07:29:34,981 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/A in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:34,981 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/1ce997309eb84ea0b2f60e8824cfada5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/45cdd05e904b4bef9212b63f81f728dd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/61994602579047d4b484691b6ec67585] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=53.4 K 2024-11-11T07:29:34,984 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ce997309eb84ea0b2f60e8824cfada5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731310171211 2024-11-11T07:29:34,987 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 45cdd05e904b4bef9212b63f81f728dd, keycount=500, bloomtype=ROW, size=28.7 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731310172982 2024-11-11T07:29:34,988 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 61994602579047d4b484691b6ec67585, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731310173078 2024-11-11T07:29:34,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-11T07:29:34,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:34,989 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-11T07:29:34,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:34,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:34,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:34,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:34,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:34,990 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:34,991 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/B is initiating minor compaction (all files) 2024-11-11T07:29:34,991 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/B in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:34,991 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/763e5c4582614d399726b1aff3b9a563, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/cc82b143f01c4d099bcf57b01d32ac54, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/933bf28a59554c88a8afc43be9e4c16a] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=34.3 K 2024-11-11T07:29:34,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:34,998 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 763e5c4582614d399726b1aff3b9a563, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731310171211 2024-11-11T07:29:35,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/480b633b5e524c9aae4bcb715a0b424a is 50, key is test_row_0/A:col10/1731310174684/Put/seqid=0 2024-11-11T07:29:35,002 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc82b143f01c4d099bcf57b01d32ac54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731310173035 2024-11-11T07:29:35,004 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 933bf28a59554c88a8afc43be9e4c16a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731310173078 2024-11-11T07:29:35,026 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#A#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:35,026 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/186775dd7d9945de88feb0e8badbe8d0 is 50, key is test_row_0/A:col10/1731310173078/Put/seqid=0 2024-11-11T07:29:35,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:35,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:35,049 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#B#compaction#71 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:35,049 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/6c2790a536b14ca6a0ccc630056690a2 is 50, key is test_row_0/B:col10/1731310173078/Put/seqid=0 2024-11-11T07:29:35,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310235064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,082 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,082 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310235066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,083 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,083 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310235067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741907_1083 (size=13085) 2024-11-11T07:29:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,087 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310235082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,091 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741906_1082 (size=12301) 2024-11-11T07:29:35,101 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/480b633b5e524c9aae4bcb715a0b424a 2024-11-11T07:29:35,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741908_1084 (size=13085) 2024-11-11T07:29:35,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/6480886905c24839ac9df6c0ed76bf3e is 50, key is test_row_0/B:col10/1731310174684/Put/seqid=0 2024-11-11T07:29:35,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741909_1085 (size=12301) 2024-11-11T07:29:35,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310235185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310235195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,207 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,215 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310235197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,218 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,219 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,218 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310235196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310235404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,406 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310235415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,417 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310235425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310235438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,444 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,445 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,498 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/186775dd7d9945de88feb0e8badbe8d0 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/186775dd7d9945de88feb0e8badbe8d0 2024-11-11T07:29:35,515 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/A of 6756dec3886a521f131ee738c3465dc1 into 186775dd7d9945de88feb0e8badbe8d0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:35,515 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:35,515 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/A, priority=13, startTime=1731310174973; duration=0sec 2024-11-11T07:29:35,516 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:35,516 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:A 2024-11-11T07:29:35,516 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:35,518 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:35,518 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/C is initiating minor compaction (all files) 2024-11-11T07:29:35,518 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/C in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:35,518 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/ad1364e1c72048c79fb62185970fe194, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/9dcb094dbd71427f8dfce24285e3cded, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6fd4af622be44a8f95e386c8f56790b6] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=34.3 K 2024-11-11T07:29:35,519 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting ad1364e1c72048c79fb62185970fe194, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=295, earliestPutTs=1731310171211 2024-11-11T07:29:35,520 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 9dcb094dbd71427f8dfce24285e3cded, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731310173035 2024-11-11T07:29:35,520 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 6fd4af622be44a8f95e386c8f56790b6, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731310173078 2024-11-11T07:29:35,543 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/6c2790a536b14ca6a0ccc630056690a2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/6c2790a536b14ca6a0ccc630056690a2 2024-11-11T07:29:35,555 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/B of 6756dec3886a521f131ee738c3465dc1 into 6c2790a536b14ca6a0ccc630056690a2(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:35,555 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:35,555 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/B, priority=13, startTime=1731310174973; duration=0sec 2024-11-11T07:29:35,555 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:35,555 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:B 2024-11-11T07:29:35,558 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#C#compaction#73 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:35,559 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/f54669a8291548f1a16ee5d32f4e7962 is 50, key is test_row_0/C:col10/1731310173078/Put/seqid=0 2024-11-11T07:29:35,575 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/6480886905c24839ac9df6c0ed76bf3e 2024-11-11T07:29:35,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741910_1086 (size=13085) 2024-11-11T07:29:35,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/d3ce650b69e94591b45b61cab65f5483 is 50, key is test_row_0/C:col10/1731310174684/Put/seqid=0 2024-11-11T07:29:35,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741911_1087 (size=12301) 2024-11-11T07:29:35,653 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=415 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/d3ce650b69e94591b45b61cab65f5483 2024-11-11T07:29:35,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/480b633b5e524c9aae4bcb715a0b424a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/480b633b5e524c9aae4bcb715a0b424a 2024-11-11T07:29:35,679 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/480b633b5e524c9aae4bcb715a0b424a, entries=150, sequenceid=415, filesize=12.0 K 2024-11-11T07:29:35,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/6480886905c24839ac9df6c0ed76bf3e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/6480886905c24839ac9df6c0ed76bf3e 2024-11-11T07:29:35,705 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/6480886905c24839ac9df6c0ed76bf3e, entries=150, sequenceid=415, filesize=12.0 K 2024-11-11T07:29:35,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/d3ce650b69e94591b45b61cab65f5483 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/d3ce650b69e94591b45b61cab65f5483 2024-11-11T07:29:35,716 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/d3ce650b69e94591b45b61cab65f5483, entries=150, sequenceid=415, filesize=12.0 K 2024-11-11T07:29:35,718 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 6756dec3886a521f131ee738c3465dc1 in 728ms, sequenceid=415, compaction requested=false 2024-11-11T07:29:35,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:35,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:35,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-11T07:29:35,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-11T07:29:35,727 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T07:29:35,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:35,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:35,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:35,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:35,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:35,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:35,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-11T07:29:35,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1510 sec 2024-11-11T07:29:35,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/cc71e716ac064ad983523ca83c07ace5 is 50, key is test_row_0/A:col10/1731310175079/Put/seqid=0 2024-11-11T07:29:35,744 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 3.1710 sec 2024-11-11T07:29:35,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741912_1088 (size=12301) 2024-11-11T07:29:35,825 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310235806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,826 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,827 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310235808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,827 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310235821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310235826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,838 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310235935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310235935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310235945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,952 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:35,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:35,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310235945, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:35,953 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:36,010 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/f54669a8291548f1a16ee5d32f4e7962 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/f54669a8291548f1a16ee5d32f4e7962 2024-11-11T07:29:36,018 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/C of 6756dec3886a521f131ee738c3465dc1 into f54669a8291548f1a16ee5d32f4e7962(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:36,019 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:36,019 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/C, priority=13, startTime=1731310174973; duration=0sec 2024-11-11T07:29:36,019 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:36,019 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:C 2024-11-11T07:29:36,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:36,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310236155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:36,158 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:36,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:36,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310236159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:36,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:36,177 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:36,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310236159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:36,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310236175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:36,184 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:36,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/cc71e716ac064ad983523ca83c07ace5 2024-11-11T07:29:36,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/061a2e03d91f41d6a54c9abbe3926673 is 50, key is test_row_0/B:col10/1731310175079/Put/seqid=0 2024-11-11T07:29:36,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741913_1089 (size=12301) 2024-11-11T07:29:36,474 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:36,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310236465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:36,475 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:36,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:36,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310236479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:36,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:36,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310236485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:36,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:36,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310236495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:36,502 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:36,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-11T07:29:36,726 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:36,726 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:29:36,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/061a2e03d91f41d6a54c9abbe3926673 2024-11-11T07:29:36,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-11T07:29:36,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T07:29:36,735 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:29:36,736 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:29:36,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:29:36,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/9a15f3a6f4204d99bc36826cf9e68703 is 50, key is test_row_0/C:col10/1731310175079/Put/seqid=0 2024-11-11T07:29:36,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741914_1090 (size=12301) 2024-11-11T07:29:36,815 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=429 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/9a15f3a6f4204d99bc36826cf9e68703 2024-11-11T07:29:36,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T07:29:36,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/cc71e716ac064ad983523ca83c07ace5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/cc71e716ac064ad983523ca83c07ace5 2024-11-11T07:29:36,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/cc71e716ac064ad983523ca83c07ace5, entries=150, sequenceid=429, filesize=12.0 K 2024-11-11T07:29:36,890 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T07:29:36,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/061a2e03d91f41d6a54c9abbe3926673 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/061a2e03d91f41d6a54c9abbe3926673 2024-11-11T07:29:36,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:36,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:36,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:36,904 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/061a2e03d91f41d6a54c9abbe3926673, entries=150, sequenceid=429, filesize=12.0 K 2024-11-11T07:29:36,905 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/9a15f3a6f4204d99bc36826cf9e68703 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/9a15f3a6f4204d99bc36826cf9e68703 2024-11-11T07:29:36,912 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/9a15f3a6f4204d99bc36826cf9e68703, entries=150, sequenceid=429, filesize=12.0 K 2024-11-11T07:29:36,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6756dec3886a521f131ee738c3465dc1 in 1186ms, sequenceid=429, compaction requested=true 2024-11-11T07:29:36,913 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:36,913 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:36,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:29:36,914 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:36,915 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:29:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:36,915 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:29:36,915 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:36,915 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/A is initiating minor compaction (all files) 2024-11-11T07:29:36,915 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/A in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:36,915 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/186775dd7d9945de88feb0e8badbe8d0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/480b633b5e524c9aae4bcb715a0b424a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/cc71e716ac064ad983523ca83c07ace5] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=36.8 K 2024-11-11T07:29:36,916 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 186775dd7d9945de88feb0e8badbe8d0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731310173043 2024-11-11T07:29:36,916 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 480b633b5e524c9aae4bcb715a0b424a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731310174667 2024-11-11T07:29:36,917 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:36,917 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/B is initiating minor compaction (all files) 2024-11-11T07:29:36,917 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/B in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:36,917 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/6c2790a536b14ca6a0ccc630056690a2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/6480886905c24839ac9df6c0ed76bf3e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/061a2e03d91f41d6a54c9abbe3926673] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=36.8 K 2024-11-11T07:29:36,917 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c2790a536b14ca6a0ccc630056690a2, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731310173043 2024-11-11T07:29:36,918 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc71e716ac064ad983523ca83c07ace5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731310175061 2024-11-11T07:29:36,918 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 6480886905c24839ac9df6c0ed76bf3e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731310174667 2024-11-11T07:29:36,919 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 061a2e03d91f41d6a54c9abbe3926673, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731310175061 2024-11-11T07:29:36,941 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#A#compaction#78 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:36,942 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/41d93c52e94f4d46ba390bfa6c2bc7e8 is 50, key is test_row_0/A:col10/1731310175079/Put/seqid=0 2024-11-11T07:29:36,949 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#B#compaction#79 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:36,950 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/a56c2ef38a504ba59aaae3094a6be39a is 50, key is test_row_0/B:col10/1731310175079/Put/seqid=0 2024-11-11T07:29:36,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310175799, max timestamp=1731310176990Num uniques 859; 2024-11-11T07:29:36,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310175799, max timestamp=1731310176990Num uniques 859; 2024-11-11T07:29:36,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:36,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:36,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310175799, max timestamp=1731310176990Num uniques 859; 2024-11-11T07:29:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:36,996 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:36,997 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=26.84 KB heapSize=66.23 KB 2024-11-11T07:29:36,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:36,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:36,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:36,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:36,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:36,997 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:37,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741915_1091 (size=13187) 2024-11-11T07:29:37,013 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/41d93c52e94f4d46ba390bfa6c2bc7e8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/41d93c52e94f4d46ba390bfa6c2bc7e8 2024-11-11T07:29:37,016 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/6fe60e9fd81b44b4995e872b4bd1381d is 50, key is test_row_0/A:col10/1731310175799/Put/seqid=0 2024-11-11T07:29:37,024 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/A of 6756dec3886a521f131ee738c3465dc1 into 41d93c52e94f4d46ba390bfa6c2bc7e8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:37,024 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:37,024 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/A, priority=13, startTime=1731310176913; duration=0sec 2024-11-11T07:29:37,024 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:37,024 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:A 2024-11-11T07:29:37,024 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:37,026 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:37,026 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/C is initiating minor compaction (all files) 2024-11-11T07:29:37,026 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/C in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:37,027 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/f54669a8291548f1a16ee5d32f4e7962, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/d3ce650b69e94591b45b61cab65f5483, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/9a15f3a6f4204d99bc36826cf9e68703] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=36.8 K 2024-11-11T07:29:37,027 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting f54669a8291548f1a16ee5d32f4e7962, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=379, earliestPutTs=1731310173043 2024-11-11T07:29:37,028 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3ce650b69e94591b45b61cab65f5483, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=415, earliestPutTs=1731310174667 2024-11-11T07:29:37,029 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a15f3a6f4204d99bc36826cf9e68703, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731310175061 2024-11-11T07:29:37,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741916_1092 (size=13187) 2024-11-11T07:29:37,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T07:29:37,062 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/a56c2ef38a504ba59aaae3094a6be39a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/a56c2ef38a504ba59aaae3094a6be39a 2024-11-11T07:29:37,063 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T07:29:37,066 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#C#compaction#81 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:37,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:37,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:37,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:37,067 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/6753ae2537b1434584eaa75afabbfef7 is 50, key is test_row_0/C:col10/1731310175079/Put/seqid=0 2024-11-11T07:29:37,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,075 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/B of 6756dec3886a521f131ee738c3465dc1 into a56c2ef38a504ba59aaae3094a6be39a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:37,075 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:37,075 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/B, priority=13, startTime=1731310176914; duration=0sec 2024-11-11T07:29:37,076 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:37,076 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:B 2024-11-11T07:29:37,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741917_1093 (size=14741) 2024-11-11T07:29:37,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/6fe60e9fd81b44b4995e872b4bd1381d 2024-11-11T07:29:37,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731310177063, max timestamp=1731310177095Num uniques 859; 2024-11-11T07:29:37,102 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T07:29:37,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310177063, max timestamp=1731310177095Num uniques 859; 2024-11-11T07:29:37,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:37,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310177063, max timestamp=1731310177095Num uniques 859; 2024-11-11T07:29:37,109 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:29:37,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/96e5e10bc5ae40b2a8f47c36eb42eeea is 50, key is test_row_0/B:col10/1731310175799/Put/seqid=0 2024-11-11T07:29:37,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741918_1094 (size=13187) 2024-11-11T07:29:37,153 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/6753ae2537b1434584eaa75afabbfef7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6753ae2537b1434584eaa75afabbfef7 2024-11-11T07:29:37,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741919_1095 (size=12301) 2024-11-11T07:29:37,178 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/C of 6756dec3886a521f131ee738c3465dc1 into 6753ae2537b1434584eaa75afabbfef7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:37,178 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:37,178 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/C, priority=13, startTime=1731310176915; duration=0sec 2024-11-11T07:29:37,178 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:37,178 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:C 2024-11-11T07:29:37,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/96e5e10bc5ae40b2a8f47c36eb42eeea 2024-11-11T07:29:37,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731310177195, max timestamp=1731310177200Num uniques 1068; 2024-11-11T07:29:37,205 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-11T07:29:37,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731310177195, max timestamp=1731310177200Num uniques 1018; 2024-11-11T07:29:37,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731310177195, max timestamp=1731310177200Num uniques 1018; 2024-11-11T07:29:37,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T07:29:37,215 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-11T07:29:37,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/18f3ad3092a041dc96534005a4a90a89 is 50, key is test_row_0/C:col10/1731310175799/Put/seqid=0 2024-11-11T07:29:37,227 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T07:29:37,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:37,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:37,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:37,227 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741920_1096 (size=12301) 2024-11-11T07:29:37,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=451 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/18f3ad3092a041dc96534005a4a90a89 2024-11-11T07:29:37,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/6fe60e9fd81b44b4995e872b4bd1381d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/6fe60e9fd81b44b4995e872b4bd1381d 2024-11-11T07:29:37,290 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/6fe60e9fd81b44b4995e872b4bd1381d, entries=200, sequenceid=451, filesize=14.4 K 2024-11-11T07:29:37,292 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/96e5e10bc5ae40b2a8f47c36eb42eeea as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/96e5e10bc5ae40b2a8f47c36eb42eeea 2024-11-11T07:29:37,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731310177245, max timestamp=1731310177284Num uniques 1027; 2024-11-11T07:29:37,294 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-11T07:29:37,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=277, cellSize=12682, totalHeapSize=29048, min timestamp=1731310177245, max timestamp=1731310177284Num uniques 1027; 2024-11-11T07:29:37,297 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=12682, suffix heap size=134776,new segment heap size=29048 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=277 2024-11-11T07:29:37,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=177, cellSize=8102, totalHeapSize=18648, min timestamp=1731310177260, max timestamp=1731310177292Num uniques 1027; 2024-11-11T07:29:37,300 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=8102, suffix heap size=134776,new segment heap size=18648 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=177 2024-11-11T07:29:37,304 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/96e5e10bc5ae40b2a8f47c36eb42eeea, entries=150, sequenceid=451, filesize=12.0 K 2024-11-11T07:29:37,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/18f3ad3092a041dc96534005a4a90a89 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/18f3ad3092a041dc96534005a4a90a89 2024-11-11T07:29:37,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/18f3ad3092a041dc96534005a4a90a89, entries=150, sequenceid=451, filesize=12.0 K 2024-11-11T07:29:37,325 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=49.20 KB/50380 for 6756dec3886a521f131ee738c3465dc1 in 321ms, sequenceid=451, compaction requested=false 2024-11-11T07:29:37,325 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:37,326 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=55.91 KB heapSize=144.63 KB 2024-11-11T07:29:37,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:37,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:37,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:37,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:37,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:37,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:37,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/7880a086d9c445e68ec7da610567ee70 is 50, key is test_row_0/A:col10/1731310177326/Put/seqid=0 2024-11-11T07:29:37,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T07:29:37,383 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T07:29:37,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:37,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:37,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:37,383 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741921_1097 (size=19621) 2024-11-11T07:29:37,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:37,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310237394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:37,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:37,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310237398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:37,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310237399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:37,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,406 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:37,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310237403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:37,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,410 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:37,523 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:37,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310237515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:37,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,527 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:37,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:37,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310237516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:37,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:37,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310237522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,529 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:37,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:37,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310237523, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:37,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:37,531 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,534 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:37,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T07:29:37,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:37,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:37,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:37,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T07:29:37,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:37,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:37,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:37,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310237736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:37,739 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:37,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310237736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:37,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:37,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310237736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:37,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,741 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:37,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:37,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310237745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:37,747 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:37,748 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:37,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/7880a086d9c445e68ec7da610567ee70 2024-11-11T07:29:37,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/0c7df2d9a34543aba2932ac8f67296b8 is 50, key is test_row_0/B:col10/1731310177326/Put/seqid=0 2024-11-11T07:29:37,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741922_1098 (size=12301) 2024-11-11T07:29:37,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/0c7df2d9a34543aba2932ac8f67296b8 2024-11-11T07:29:37,859 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T07:29:37,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:37,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:37,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:37,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:37,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T07:29:37,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/acac6f07a400443d81bd525bdde833f6 is 50, key is test_row_0/C:col10/1731310177326/Put/seqid=0 2024-11-11T07:29:37,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741923_1099 (size=12301) 2024-11-11T07:29:37,941 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=514 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/acac6f07a400443d81bd525bdde833f6 2024-11-11T07:29:37,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/7880a086d9c445e68ec7da610567ee70 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/7880a086d9c445e68ec7da610567ee70 2024-11-11T07:29:37,960 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/7880a086d9c445e68ec7da610567ee70, entries=300, sequenceid=514, filesize=19.2 K 2024-11-11T07:29:37,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/0c7df2d9a34543aba2932ac8f67296b8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/0c7df2d9a34543aba2932ac8f67296b8 2024-11-11T07:29:37,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/0c7df2d9a34543aba2932ac8f67296b8, entries=150, sequenceid=514, filesize=12.0 K 2024-11-11T07:29:37,976 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/acac6f07a400443d81bd525bdde833f6 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/acac6f07a400443d81bd525bdde833f6 2024-11-11T07:29:37,983 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/acac6f07a400443d81bd525bdde833f6, entries=150, sequenceid=514, filesize=12.0 K 2024-11-11T07:29:37,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~69.33 KB/70990, heapSize ~184.42 KB/188848, currentSize=114.05 KB/116790 for 6756dec3886a521f131ee738c3465dc1 in 660ms, sequenceid=514, compaction requested=true 2024-11-11T07:29:37,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:37,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:29:37,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:37,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:29:37,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:29:37,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:29:37,987 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T07:29:37,987 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:37,987 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:37,989 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:37,989 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 47549 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:37,989 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/C is initiating minor compaction (all files) 2024-11-11T07:29:37,989 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/A is initiating minor compaction (all files) 2024-11-11T07:29:37,989 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/C in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:37,989 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/A in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:37,989 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/41d93c52e94f4d46ba390bfa6c2bc7e8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/6fe60e9fd81b44b4995e872b4bd1381d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/7880a086d9c445e68ec7da610567ee70] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=46.4 K 2024-11-11T07:29:37,989 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6753ae2537b1434584eaa75afabbfef7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/18f3ad3092a041dc96534005a4a90a89, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/acac6f07a400443d81bd525bdde833f6] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=36.9 K 2024-11-11T07:29:37,990 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41d93c52e94f4d46ba390bfa6c2bc7e8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731310175061 2024-11-11T07:29:37,991 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 6753ae2537b1434584eaa75afabbfef7, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731310175061 2024-11-11T07:29:37,991 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fe60e9fd81b44b4995e872b4bd1381d, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731310175799 2024-11-11T07:29:37,991 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 18f3ad3092a041dc96534005a4a90a89, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731310175799 2024-11-11T07:29:37,992 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting acac6f07a400443d81bd525bdde833f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731310177260 2024-11-11T07:29:37,992 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7880a086d9c445e68ec7da610567ee70, keycount=300, bloomtype=ROW, size=19.2 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731310177260 2024-11-11T07:29:38,012 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#A#compaction#87 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:38,012 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/ce392774c7ab4ecda1e8e43a871939d7 is 50, key is test_row_0/A:col10/1731310177326/Put/seqid=0 2024-11-11T07:29:38,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-11T07:29:38,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:38,017 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:29:38,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:38,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:38,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:38,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:38,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:38,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:38,032 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#C#compaction#88 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:38,033 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/8f83bb02104e4d1bb922c655f25bd843 is 50, key is test_row_0/C:col10/1731310177326/Put/seqid=0 2024-11-11T07:29:38,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:38,047 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:38,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/d6f1ebdedb7947149c11bec2802c9dbc is 50, key is test_row_0/A:col10/1731310177392/Put/seqid=0 2024-11-11T07:29:38,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741924_1100 (size=13289) 2024-11-11T07:29:38,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:38,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310238079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:38,095 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:38,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,096 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:38,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310238079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:38,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,096 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:38,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:38,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310238082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:38,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:38,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:38,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310238083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:38,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741925_1101 (size=13289) 2024-11-11T07:29:38,131 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/8f83bb02104e4d1bb922c655f25bd843 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/8f83bb02104e4d1bb922c655f25bd843 2024-11-11T07:29:38,144 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/C of 6756dec3886a521f131ee738c3465dc1 into 8f83bb02104e4d1bb922c655f25bd843(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:38,145 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:38,145 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/C, priority=13, startTime=1731310177987; duration=0sec 2024-11-11T07:29:38,145 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:38,145 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:C 2024-11-11T07:29:38,145 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:38,147 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:38,147 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/B is initiating minor compaction (all files) 2024-11-11T07:29:38,147 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/B in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:38,148 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/a56c2ef38a504ba59aaae3094a6be39a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/96e5e10bc5ae40b2a8f47c36eb42eeea, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/0c7df2d9a34543aba2932ac8f67296b8] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=36.9 K 2024-11-11T07:29:38,148 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting a56c2ef38a504ba59aaae3094a6be39a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=429, earliestPutTs=1731310175061 2024-11-11T07:29:38,149 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 96e5e10bc5ae40b2a8f47c36eb42eeea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=451, earliestPutTs=1731310175799 2024-11-11T07:29:38,150 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c7df2d9a34543aba2932ac8f67296b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731310177260 2024-11-11T07:29:38,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741926_1102 (size=12301) 2024-11-11T07:29:38,166 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#B#compaction#90 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:38,167 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/4e99151ca20846d69006b3f980e469b7 is 50, key is test_row_0/B:col10/1731310177326/Put/seqid=0 2024-11-11T07:29:38,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741927_1103 (size=13289) 2024-11-11T07:29:38,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:38,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310238205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:38,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:38,217 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/4e99151ca20846d69006b3f980e469b7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/4e99151ca20846d69006b3f980e469b7 2024-11-11T07:29:38,225 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/B of 6756dec3886a521f131ee738c3465dc1 into 4e99151ca20846d69006b3f980e469b7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:38,226 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:38,226 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/B, priority=13, startTime=1731310177987; duration=0sec 2024-11-11T07:29:38,225 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:38,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310238214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:38,226 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:38,226 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:B 2024-11-11T07:29:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,226 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:38,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310238214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:38,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,227 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:38,227 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:38,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310238221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:38,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,228 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:38,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310238425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:38,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310238435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:38,444 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310238435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:38,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310238435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:38,490 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/ce392774c7ab4ecda1e8e43a871939d7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/ce392774c7ab4ecda1e8e43a871939d7 2024-11-11T07:29:38,498 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/A of 6756dec3886a521f131ee738c3465dc1 into ce392774c7ab4ecda1e8e43a871939d7(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:38,498 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:38,498 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/A, priority=13, startTime=1731310177987; duration=0sec 2024-11-11T07:29:38,499 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:38,499 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:A 2024-11-11T07:29:38,555 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/d6f1ebdedb7947149c11bec2802c9dbc 2024-11-11T07:29:38,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/ddc3635c643c4a0da2912211b470273d is 50, key is test_row_0/B:col10/1731310177392/Put/seqid=0 2024-11-11T07:29:38,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741928_1104 (size=12301) 2024-11-11T07:29:38,675 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/ddc3635c643c4a0da2912211b470273d 2024-11-11T07:29:38,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/aa34c444e3594db68fc698034c48cf97 is 50, key is test_row_0/C:col10/1731310177392/Put/seqid=0 2024-11-11T07:29:38,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741929_1105 (size=12301) 2024-11-11T07:29:38,721 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/aa34c444e3594db68fc698034c48cf97 2024-11-11T07:29:38,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/d6f1ebdedb7947149c11bec2802c9dbc as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/d6f1ebdedb7947149c11bec2802c9dbc 2024-11-11T07:29:38,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:38,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310238735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:38,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,745 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:38,753 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/d6f1ebdedb7947149c11bec2802c9dbc, entries=150, sequenceid=534, filesize=12.0 K 2024-11-11T07:29:38,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/ddc3635c643c4a0da2912211b470273d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ddc3635c643c4a0da2912211b470273d 2024-11-11T07:29:38,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:38,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310238755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:38,760 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,761 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:38,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:38,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310238755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:38,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:38,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310238755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:38,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,762 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:38,764 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:38,768 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ddc3635c643c4a0da2912211b470273d, entries=150, sequenceid=534, filesize=12.0 K 2024-11-11T07:29:38,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/aa34c444e3594db68fc698034c48cf97 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/aa34c444e3594db68fc698034c48cf97 2024-11-11T07:29:38,801 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/aa34c444e3594db68fc698034c48cf97, entries=150, sequenceid=534, filesize=12.0 K 2024-11-11T07:29:38,802 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 6756dec3886a521f131ee738c3465dc1 in 785ms, sequenceid=534, compaction requested=false 2024-11-11T07:29:38,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:38,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:38,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-11T07:29:38,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-11T07:29:38,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-11T07:29:38,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0690 sec 2024-11-11T07:29:38,810 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 2.0830 sec 2024-11-11T07:29:38,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-11T07:29:38,887 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:38,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:29:38,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-11T07:29:38,891 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:29:38,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T07:29:38,892 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:29:38,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:29:38,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T07:29:39,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-11T07:29:39,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:39,046 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T07:29:39,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:39,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:39,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:39,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:39,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:39,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:39,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/72b1f346241e442787f88ecb5d3bc002 is 50, key is test_row_0/A:col10/1731310178076/Put/seqid=0 2024-11-11T07:29:39,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741930_1106 (size=12301) 2024-11-11T07:29:39,108 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/72b1f346241e442787f88ecb5d3bc002 2024-11-11T07:29:39,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/cb79caa35f1d4ae49ec99c48f27b5e01 is 50, key is test_row_0/B:col10/1731310178076/Put/seqid=0 2024-11-11T07:29:39,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741931_1107 (size=12301) 2024-11-11T07:29:39,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T07:29:39,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:39,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:39,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:39,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310239315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:39,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:39,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310239329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:39,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,347 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:39,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,351 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:39,352 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:39,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310239348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:39,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:39,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310239348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,353 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:39,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:39,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310239457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:39,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:39,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310239459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,467 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:39,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:39,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310239464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:39,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310239464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:39,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T07:29:39,571 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/cb79caa35f1d4ae49ec99c48f27b5e01 2024-11-11T07:29:39,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/2f5c39dcc84f4c15841773f5946d10d3 is 50, key is test_row_0/C:col10/1731310178076/Put/seqid=0 2024-11-11T07:29:39,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741932_1108 (size=12301) 2024-11-11T07:29:39,613 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=550 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/2f5c39dcc84f4c15841773f5946d10d3 2024-11-11T07:29:39,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/72b1f346241e442787f88ecb5d3bc002 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/72b1f346241e442787f88ecb5d3bc002 2024-11-11T07:29:39,641 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/72b1f346241e442787f88ecb5d3bc002, entries=150, sequenceid=550, filesize=12.0 K 2024-11-11T07:29:39,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/cb79caa35f1d4ae49ec99c48f27b5e01 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/cb79caa35f1d4ae49ec99c48f27b5e01 2024-11-11T07:29:39,648 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/cb79caa35f1d4ae49ec99c48f27b5e01, entries=150, sequenceid=550, filesize=12.0 K 2024-11-11T07:29:39,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.StoreScanner(997): StoreScanner already has the close lock. There is no need to updateReaders 2024-11-11T07:29:39,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/2f5c39dcc84f4c15841773f5946d10d3 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/2f5c39dcc84f4c15841773f5946d10d3 2024-11-11T07:29:39,667 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/2f5c39dcc84f4c15841773f5946d10d3, entries=150, sequenceid=550, filesize=12.0 K 2024-11-11T07:29:39,668 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6756dec3886a521f131ee738c3465dc1 in 622ms, sequenceid=550, compaction requested=true 2024-11-11T07:29:39,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:39,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:39,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-11T07:29:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:39,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-11T07:29:39,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:29:39,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-11T07:29:39,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 787 msec 2024-11-11T07:29:39,684 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 794 msec 2024-11-11T07:29:39,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:39,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:39,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:39,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:39,687 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:39,688 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:39,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/3ac9c8726b3245b0ac8c81f6d5ed2a2e is 50, key is test_row_0/A:col10/1731310179311/Put/seqid=0 2024-11-11T07:29:39,713 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:39,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310239703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:39,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,714 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:39,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310239710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:39,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:39,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,720 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:39,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310239710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:39,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:39,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:39,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310239712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:39,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,723 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:39,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741933_1109 (size=14741) 2024-11-11T07:29:39,740 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/3ac9c8726b3245b0ac8c81f6d5ed2a2e 2024-11-11T07:29:39,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/fa0725b1361745908f245f4a359db629 is 50, key is test_row_0/B:col10/1731310179311/Put/seqid=0 2024-11-11T07:29:39,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741934_1110 (size=12301) 2024-11-11T07:29:39,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310239824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:39,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:39,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310239825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:39,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310239825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:39,830 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,831 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:39,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:39,831 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310239825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:39,832 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:40,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-11T07:29:40,025 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:40,026 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:29:40,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-11T07:29:40,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T07:29:40,028 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:29:40,030 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:29:40,030 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:29:40,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:40,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310240035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310240035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:40,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310240035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:40,050 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:40,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310240035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,050 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:40,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T07:29:40,168 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/fa0725b1361745908f245f4a359db629 2024-11-11T07:29:40,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/b559002437844fb483c09fd0ba265ce3 is 50, key is test_row_0/C:col10/1731310179311/Put/seqid=0 2024-11-11T07:29:40,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T07:29:40,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:40,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:40,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:40,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:40,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:40,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:40,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741935_1111 (size=12301) 2024-11-11T07:29:40,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T07:29:40,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T07:29:40,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:40,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:40,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:40,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:40,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:40,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:40,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:40,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310240355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,365 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:40,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:40,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310240355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:40,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,366 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310240355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:40,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:40,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310240356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,367 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,368 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:40,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T07:29:40,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:40,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:40,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:40,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:40,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:40,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:40,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/b559002437844fb483c09fd0ba265ce3 2024-11-11T07:29:40,653 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/3ac9c8726b3245b0ac8c81f6d5ed2a2e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/3ac9c8726b3245b0ac8c81f6d5ed2a2e 2024-11-11T07:29:40,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T07:29:40,660 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/3ac9c8726b3245b0ac8c81f6d5ed2a2e, entries=200, sequenceid=571, filesize=14.4 K 2024-11-11T07:29:40,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/fa0725b1361745908f245f4a359db629 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/fa0725b1361745908f245f4a359db629 2024-11-11T07:29:40,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T07:29:40,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:40,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:40,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:40,670 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:40,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:40,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:40,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/fa0725b1361745908f245f4a359db629, entries=150, sequenceid=571, filesize=12.0 K 2024-11-11T07:29:40,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/b559002437844fb483c09fd0ba265ce3 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/b559002437844fb483c09fd0ba265ce3 2024-11-11T07:29:40,704 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/b559002437844fb483c09fd0ba265ce3, entries=150, sequenceid=571, filesize=12.0 K 2024-11-11T07:29:40,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 6756dec3886a521f131ee738c3465dc1 in 1027ms, sequenceid=571, compaction requested=true 2024-11-11T07:29:40,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:40,706 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:29:40,707 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52632 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:29:40,707 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/A is initiating minor compaction (all files) 2024-11-11T07:29:40,707 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/A in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:40,708 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/ce392774c7ab4ecda1e8e43a871939d7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/d6f1ebdedb7947149c11bec2802c9dbc, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/72b1f346241e442787f88ecb5d3bc002, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/3ac9c8726b3245b0ac8c81f6d5ed2a2e] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=51.4 K 2024-11-11T07:29:40,709 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting ce392774c7ab4ecda1e8e43a871939d7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731310177260 2024-11-11T07:29:40,709 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting d6f1ebdedb7947149c11bec2802c9dbc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731310177392 2024-11-11T07:29:40,710 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 72b1f346241e442787f88ecb5d3bc002, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1731310178076 2024-11-11T07:29:40,710 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ac9c8726b3245b0ac8c81f6d5ed2a2e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731310179311 2024-11-11T07:29:40,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:29:40,719 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:40,719 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:29:40,720 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:29:40,720 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/B is initiating minor compaction (all files) 2024-11-11T07:29:40,721 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/B in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:40,721 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/4e99151ca20846d69006b3f980e469b7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ddc3635c643c4a0da2912211b470273d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/cb79caa35f1d4ae49ec99c48f27b5e01, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/fa0725b1361745908f245f4a359db629] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=49.0 K 2024-11-11T07:29:40,721 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e99151ca20846d69006b3f980e469b7, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731310177260 2024-11-11T07:29:40,722 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddc3635c643c4a0da2912211b470273d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731310177392 2024-11-11T07:29:40,722 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb79caa35f1d4ae49ec99c48f27b5e01, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1731310178076 2024-11-11T07:29:40,722 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa0725b1361745908f245f4a359db629, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731310179311 2024-11-11T07:29:40,730 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:29:40,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:40,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:29:40,731 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:40,748 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#A#compaction#99 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:40,748 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/6417a5ee06c949f894dea6ee0841d1c5 is 50, key is test_row_0/A:col10/1731310179311/Put/seqid=0 2024-11-11T07:29:40,761 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#B#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:40,762 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/f6abdbece507441d82dde0b3d93d9d00 is 50, key is test_row_0/B:col10/1731310179311/Put/seqid=0 2024-11-11T07:29:40,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741936_1112 (size=13425) 2024-11-11T07:29:40,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741937_1113 (size=13425) 2024-11-11T07:29:40,802 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/6417a5ee06c949f894dea6ee0841d1c5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/6417a5ee06c949f894dea6ee0841d1c5 2024-11-11T07:29:40,812 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/f6abdbece507441d82dde0b3d93d9d00 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/f6abdbece507441d82dde0b3d93d9d00 2024-11-11T07:29:40,814 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/A of 6756dec3886a521f131ee738c3465dc1 into 6417a5ee06c949f894dea6ee0841d1c5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:40,814 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:40,814 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/A, priority=12, startTime=1731310180705; duration=0sec 2024-11-11T07:29:40,814 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:40,814 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:A 2024-11-11T07:29:40,815 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:29:40,819 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:29:40,819 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/C is initiating minor compaction (all files) 2024-11-11T07:29:40,819 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/C in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:40,819 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/8f83bb02104e4d1bb922c655f25bd843, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/aa34c444e3594db68fc698034c48cf97, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/2f5c39dcc84f4c15841773f5946d10d3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/b559002437844fb483c09fd0ba265ce3] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=49.0 K 2024-11-11T07:29:40,820 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f83bb02104e4d1bb922c655f25bd843, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=514, earliestPutTs=1731310177260 2024-11-11T07:29:40,820 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting aa34c444e3594db68fc698034c48cf97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731310177392 2024-11-11T07:29:40,821 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f5c39dcc84f4c15841773f5946d10d3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=550, earliestPutTs=1731310178076 2024-11-11T07:29:40,821 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting b559002437844fb483c09fd0ba265ce3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731310179311 2024-11-11T07:29:40,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-11T07:29:40,829 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/B of 6756dec3886a521f131ee738c3465dc1 into f6abdbece507441d82dde0b3d93d9d00(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:40,829 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:40,829 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/B, priority=12, startTime=1731310180719; duration=0sec 2024-11-11T07:29:40,830 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:40,830 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:B 2024-11-11T07:29:40,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:40,831 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T07:29:40,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:40,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:40,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:40,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:40,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:40,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:40,840 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#C#compaction#101 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:40,840 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/06934fc46b62464f840f7ea38c9d507d is 50, key is test_row_0/C:col10/1731310179311/Put/seqid=0 2024-11-11T07:29:40,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/4e79c6f1a9b845d18f0b1fe8c70d397c is 50, key is test_row_0/A:col10/1731310179702/Put/seqid=0 2024-11-11T07:29:40,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:40,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. as already flushing 2024-11-11T07:29:40,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741939_1115 (size=12301) 2024-11-11T07:29:40,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741938_1114 (size=13425) 2024-11-11T07:29:40,885 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/4e79c6f1a9b845d18f0b1fe8c70d397c 2024-11-11T07:29:40,904 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/06934fc46b62464f840f7ea38c9d507d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/06934fc46b62464f840f7ea38c9d507d 2024-11-11T07:29:40,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/2dd6a29287b24903bf1d233e892c7b97 is 50, key is test_row_0/B:col10/1731310179702/Put/seqid=0 2024-11-11T07:29:40,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741940_1116 (size=12301) 2024-11-11T07:29:40,923 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/C of 6756dec3886a521f131ee738c3465dc1 into 06934fc46b62464f840f7ea38c9d507d(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:40,923 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:40,923 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/C, priority=12, startTime=1731310180731; duration=0sec 2024-11-11T07:29:40,923 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:40,923 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:C 2024-11-11T07:29:40,936 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:40,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310240926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,937 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:40,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310240927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:40,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:40,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310240935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:40,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:40,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,938 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,939 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:40,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310240937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:40,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:40,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:41,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310241045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:41,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:41,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310241045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:41,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:41,050 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310241045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:41,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310241045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,053 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:41,051 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:41,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T07:29:41,258 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:41,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310241255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,259 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:41,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:41,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310241265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:41,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:41,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310241266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310241265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,270 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:41,328 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/2dd6a29287b24903bf1d233e892c7b97 2024-11-11T07:29:41,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/f3c9153db89e4c969f8272de7c0e67f9 is 50, key is test_row_0/C:col10/1731310179702/Put/seqid=0 2024-11-11T07:29:41,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741941_1117 (size=12301) 2024-11-11T07:29:41,564 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-11T07:29:41,565 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:29:41,568 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-11T07:29:41,568 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:29:41,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:41,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34324 deadline: 1731310241567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:41,571 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-11T07:29:41,571 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:29:41,571 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:41,571 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:41,571 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:41,571 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:41,571 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:29:41,572 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:41,572 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:29:41,573 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:41,573 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:29:41,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:41,575 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34304 deadline: 1731310241574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:41,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34298 deadline: 1731310241575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:41,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,576 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:41,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,576 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:41,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:41,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34316 deadline: 1731310241575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:41,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,576 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:41,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:29:41,579 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-11T07:29:41,579 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:29:41,579 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:41,579 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:41,579 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:29:41,779 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=584 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/f3c9153db89e4c969f8272de7c0e67f9 2024-11-11T07:29:41,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/4e79c6f1a9b845d18f0b1fe8c70d397c as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/4e79c6f1a9b845d18f0b1fe8c70d397c 2024-11-11T07:29:41,791 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/4e79c6f1a9b845d18f0b1fe8c70d397c, entries=150, sequenceid=584, filesize=12.0 K 2024-11-11T07:29:41,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/2dd6a29287b24903bf1d233e892c7b97 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/2dd6a29287b24903bf1d233e892c7b97 2024-11-11T07:29:41,798 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/2dd6a29287b24903bf1d233e892c7b97, entries=150, sequenceid=584, filesize=12.0 K 2024-11-11T07:29:41,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/f3c9153db89e4c969f8272de7c0e67f9 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/f3c9153db89e4c969f8272de7c0e67f9 2024-11-11T07:29:41,805 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/f3c9153db89e4c969f8272de7c0e67f9, entries=150, sequenceid=584, filesize=12.0 K 2024-11-11T07:29:41,806 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=120.76 KB/123660 for 6756dec3886a521f131ee738c3465dc1 in 975ms, sequenceid=584, compaction requested=false 2024-11-11T07:29:41,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:41,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:41,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-11T07:29:41,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-11T07:29:41,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-11T07:29:41,810 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7780 sec 2024-11-11T07:29:41,812 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 1.7850 sec 2024-11-11T07:29:42,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:42,076 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=127.47 KB heapSize=379.97 KB 2024-11-11T07:29:42,076 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-11T07:29:42,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:42,076 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:29:42,076 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:42,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:42,076 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:42,076 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:42,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:42,077 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:29:42,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:42,077 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:42,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/da72250a6b804265bc5b3f5b4b73587c is 50, key is test_row_0/A:col10/1731310180911/Put/seqid=0 2024-11-11T07:29:42,087 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-11T07:29:42,087 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-11T07:29:42,087 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:29:42,087 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:29:42,087 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:42,087 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:42,087 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:42,087 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:42,087 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:29:42,087 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:29:42,088 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-11T07:29:42,088 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:29:42,088 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:42,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:29:42,088 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:42,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741942_1118 (size=12301) 2024-11-11T07:29:42,093 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/da72250a6b804265bc5b3f5b4b73587c 2024-11-11T07:29:42,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/7bfe43dfd6b942e99993cd3e78f17499 is 50, key is test_row_0/B:col10/1731310180911/Put/seqid=0 2024-11-11T07:29:42,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741943_1119 (size=12301) 2024-11-11T07:29:42,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/7bfe43dfd6b942e99993cd3e78f17499 2024-11-11T07:29:42,117 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/90c6b73f98584401a5707d6d950a1477 is 50, key is test_row_0/C:col10/1731310180911/Put/seqid=0 2024-11-11T07:29:42,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741944_1120 (size=12301) 2024-11-11T07:29:42,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=607 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/90c6b73f98584401a5707d6d950a1477 2024-11-11T07:29:42,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/da72250a6b804265bc5b3f5b4b73587c as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/da72250a6b804265bc5b3f5b4b73587c 2024-11-11T07:29:42,140 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/da72250a6b804265bc5b3f5b4b73587c, entries=150, sequenceid=607, filesize=12.0 K 2024-11-11T07:29:42,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/7bfe43dfd6b942e99993cd3e78f17499 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/7bfe43dfd6b942e99993cd3e78f17499 2024-11-11T07:29:42,148 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/7bfe43dfd6b942e99993cd3e78f17499, entries=150, sequenceid=607, filesize=12.0 K 2024-11-11T07:29:42,150 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/90c6b73f98584401a5707d6d950a1477 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/90c6b73f98584401a5707d6d950a1477 2024-11-11T07:29:42,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/90c6b73f98584401a5707d6d950a1477, entries=150, sequenceid=607, filesize=12.0 K 2024-11-11T07:29:42,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~379.92 KB/389040, currentSize=20.13 KB/20610 for 6756dec3886a521f131ee738c3465dc1 in 82ms, sequenceid=607, compaction requested=true 2024-11-11T07:29:42,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:42,158 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:42,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:29:42,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:42,159 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:42,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:29:42,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:42,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6756dec3886a521f131ee738c3465dc1:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:29:42,159 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:42,160 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:42,160 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:42,160 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/A is initiating minor compaction (all files) 2024-11-11T07:29:42,160 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/B is initiating minor compaction (all files) 2024-11-11T07:29:42,160 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/A in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:42,160 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/B in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:42,161 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/6417a5ee06c949f894dea6ee0841d1c5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/4e79c6f1a9b845d18f0b1fe8c70d397c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/da72250a6b804265bc5b3f5b4b73587c] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=37.1 K 2024-11-11T07:29:42,161 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/f6abdbece507441d82dde0b3d93d9d00, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/2dd6a29287b24903bf1d233e892c7b97, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/7bfe43dfd6b942e99993cd3e78f17499] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=37.1 K 2024-11-11T07:29:42,162 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f6abdbece507441d82dde0b3d93d9d00, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731310179311 2024-11-11T07:29:42,162 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6417a5ee06c949f894dea6ee0841d1c5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731310179311 2024-11-11T07:29:42,162 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 2dd6a29287b24903bf1d233e892c7b97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731310179692 2024-11-11T07:29:42,163 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e79c6f1a9b845d18f0b1fe8c70d397c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731310179692 2024-11-11T07:29:42,163 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bfe43dfd6b942e99993cd3e78f17499, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1731310180911 2024-11-11T07:29:42,164 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting da72250a6b804265bc5b3f5b4b73587c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1731310180911 2024-11-11T07:29:42,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-11T07:29:42,176 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:42,188 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#B#compaction#108 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:42,189 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/ba7be5fdc9a749abac5def58c574980d is 50, key is test_row_0/B:col10/1731310180911/Put/seqid=0 2024-11-11T07:29:42,194 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#A#compaction#109 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:42,194 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/7bca83ac2efd4a6ba519a6d42e3c0606 is 50, key is test_row_0/A:col10/1731310180911/Put/seqid=0 2024-11-11T07:29:42,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741945_1121 (size=13527) 2024-11-11T07:29:42,210 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/ba7be5fdc9a749abac5def58c574980d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ba7be5fdc9a749abac5def58c574980d 2024-11-11T07:29:42,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741946_1122 (size=13527) 2024-11-11T07:29:42,218 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/B of 6756dec3886a521f131ee738c3465dc1 into ba7be5fdc9a749abac5def58c574980d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:42,218 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:42,218 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/B, priority=13, startTime=1731310182159; duration=0sec 2024-11-11T07:29:42,219 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:42,219 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:B 2024-11-11T07:29:42,219 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:42,220 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:42,220 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6756dec3886a521f131ee738c3465dc1/C is initiating minor compaction (all files) 2024-11-11T07:29:42,220 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6756dec3886a521f131ee738c3465dc1/C in TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:42,220 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/06934fc46b62464f840f7ea38c9d507d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/f3c9153db89e4c969f8272de7c0e67f9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/90c6b73f98584401a5707d6d950a1477] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp, totalSize=37.1 K 2024-11-11T07:29:42,221 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 06934fc46b62464f840f7ea38c9d507d, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731310179311 2024-11-11T07:29:42,221 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f3c9153db89e4c969f8272de7c0e67f9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=584, earliestPutTs=1731310179692 2024-11-11T07:29:42,222 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 90c6b73f98584401a5707d6d950a1477, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=607, earliestPutTs=1731310180911 2024-11-11T07:29:42,236 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6756dec3886a521f131ee738c3465dc1#C#compaction#110 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:42,237 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/f2aa2be27f2a4b6489e24f76054a2f77 is 50, key is test_row_0/C:col10/1731310180911/Put/seqid=0 2024-11-11T07:29:42,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741947_1123 (size=13527) 2024-11-11T07:29:42,259 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/f2aa2be27f2a4b6489e24f76054a2f77 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/f2aa2be27f2a4b6489e24f76054a2f77 2024-11-11T07:29:42,266 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/C of 6756dec3886a521f131ee738c3465dc1 into f2aa2be27f2a4b6489e24f76054a2f77(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:42,266 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:42,266 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/C, priority=13, startTime=1731310182159; duration=0sec 2024-11-11T07:29:42,266 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:42,266 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:C 2024-11-11T07:29:42,620 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/7bca83ac2efd4a6ba519a6d42e3c0606 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/7bca83ac2efd4a6ba519a6d42e3c0606 2024-11-11T07:29:42,626 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6756dec3886a521f131ee738c3465dc1/A of 6756dec3886a521f131ee738c3465dc1 into 7bca83ac2efd4a6ba519a6d42e3c0606(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:42,626 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6756dec3886a521f131ee738c3465dc1: 2024-11-11T07:29:42,626 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., storeName=6756dec3886a521f131ee738c3465dc1/A, priority=13, startTime=1731310182158; duration=0sec 2024-11-11T07:29:42,627 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:42,627 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6756dec3886a521f131ee738c3465dc1:A 2024-11-11T07:29:43,334 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T07:29:44,846 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-11T07:29:44,846 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:29:44,846 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:44,847 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:44,847 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:29:44,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T07:29:44,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-11T07:29:44,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-11T07:29:44,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-11T07:29:44,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-11-11T07:29:44,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 25 2024-11-11T07:29:44,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T07:29:44,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1247 2024-11-11T07:29:44,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1252 2024-11-11T07:29:44,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T07:29:44,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 618 2024-11-11T07:29:44,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1830 rows 2024-11-11T07:29:44,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 609 2024-11-11T07:29:44,847 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1826 rows 2024-11-11T07:29:44,848 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T07:29:44,848 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:29:44,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:44,848 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:44,848 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:29:44,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T07:29:44,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T07:29:44,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T07:29:44,862 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310184862"}]},"ts":"1731310184862"} 2024-11-11T07:29:44,864 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T07:29:44,864 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T07:29:44,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:29:44,873 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6756dec3886a521f131ee738c3465dc1, UNASSIGN}] 2024-11-11T07:29:44,874 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6756dec3886a521f131ee738c3465dc1, UNASSIGN 2024-11-11T07:29:44,880 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=6756dec3886a521f131ee738c3465dc1, regionState=CLOSING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:44,886 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6756dec3886a521f131ee738c3465dc1, UNASSIGN because future has completed 2024-11-11T07:29:44,887 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:29:44,887 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:29:44,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T07:29:45,045 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:45,045 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:29:45,046 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 6756dec3886a521f131ee738c3465dc1, disabling compactions & flushes 2024-11-11T07:29:45,046 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:45,046 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:45,046 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. after waiting 0 ms 2024-11-11T07:29:45,046 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:45,046 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 6756dec3886a521f131ee738c3465dc1 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-11T07:29:45,046 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=A 2024-11-11T07:29:45,046 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:45,046 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=B 2024-11-11T07:29:45,046 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:45,047 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6756dec3886a521f131ee738c3465dc1, store=C 2024-11-11T07:29:45,047 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:45,050 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/3baaf1be5575412f8b7ba2ccdea80064 is 50, key is test_row_1/A:col10/1731310182085/Put/seqid=0 2024-11-11T07:29:45,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741948_1124 (size=9857) 2024-11-11T07:29:45,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T07:29:45,455 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/3baaf1be5575412f8b7ba2ccdea80064 2024-11-11T07:29:45,465 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/64097deb5c2e473aabfce3f9c1e6957a is 50, key is test_row_1/B:col10/1731310182085/Put/seqid=0 2024-11-11T07:29:45,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741949_1125 (size=9857) 2024-11-11T07:29:45,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T07:29:45,872 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/64097deb5c2e473aabfce3f9c1e6957a 2024-11-11T07:29:45,883 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/e4170eba32304ce09c3470570220b850 is 50, key is test_row_1/C:col10/1731310182085/Put/seqid=0 2024-11-11T07:29:45,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741950_1126 (size=9857) 2024-11-11T07:29:45,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T07:29:46,308 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=617 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/e4170eba32304ce09c3470570220b850 2024-11-11T07:29:46,319 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/A/3baaf1be5575412f8b7ba2ccdea80064 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/3baaf1be5575412f8b7ba2ccdea80064 2024-11-11T07:29:46,326 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/3baaf1be5575412f8b7ba2ccdea80064, entries=100, sequenceid=617, filesize=9.6 K 2024-11-11T07:29:46,328 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/B/64097deb5c2e473aabfce3f9c1e6957a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/64097deb5c2e473aabfce3f9c1e6957a 2024-11-11T07:29:46,335 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/64097deb5c2e473aabfce3f9c1e6957a, entries=100, sequenceid=617, filesize=9.6 K 2024-11-11T07:29:46,337 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/.tmp/C/e4170eba32304ce09c3470570220b850 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/e4170eba32304ce09c3470570220b850 2024-11-11T07:29:46,345 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/e4170eba32304ce09c3470570220b850, entries=100, sequenceid=617, filesize=9.6 K 2024-11-11T07:29:46,367 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 6756dec3886a521f131ee738c3465dc1 in 1321ms, sequenceid=617, compaction requested=false 2024-11-11T07:29:46,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/dd5b766af4f246b8a140d6ef38ac589b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/0c0583b3a8f3421dbc04b1f391213259, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/347541acc3cb4c36bf671748aa341172, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/eb707fd6a6e6474faea26229db2d87f9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/56d8fd37ad974ae18218b2ecb6fa2ed5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/e01afd75d7bd4433a72186b489374df7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/58c1fd5a7f5b4a49aee087d179c91c9e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/2630135164e74bb19d74239b7bfffc34, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/581701ef4029468d8fb3079cd1c32c61, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/57e2f6d5db3c4c43b989f3104736be97, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/d557d71835e54db19cdced17395dda7a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/f4794c295d7f4e40a9dbe4258a117ac1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/316011ad1d8c43f38fd390b7976d9b34, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/7632bdb3c1fe418fbbca5abb6ef753b4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/26def2790f794d4694191a84c32aa3e3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/0399fd0f411b4d0bbda6ae7532947c90, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/fe69164e57c64fb2a5a6c0c2c2f130d1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/9b525c81b620403aab11dbd302791120, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/c812fd10ce444f6bac0264ec2029eb4a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/371429fd1da64bc0b2507cd703f215ce, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/1ce997309eb84ea0b2f60e8824cfada5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/45cdd05e904b4bef9212b63f81f728dd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/186775dd7d9945de88feb0e8badbe8d0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/61994602579047d4b484691b6ec67585, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/480b633b5e524c9aae4bcb715a0b424a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/41d93c52e94f4d46ba390bfa6c2bc7e8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/cc71e716ac064ad983523ca83c07ace5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/6fe60e9fd81b44b4995e872b4bd1381d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/7880a086d9c445e68ec7da610567ee70, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/ce392774c7ab4ecda1e8e43a871939d7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/d6f1ebdedb7947149c11bec2802c9dbc, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/72b1f346241e442787f88ecb5d3bc002, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/3ac9c8726b3245b0ac8c81f6d5ed2a2e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/6417a5ee06c949f894dea6ee0841d1c5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/4e79c6f1a9b845d18f0b1fe8c70d397c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/da72250a6b804265bc5b3f5b4b73587c] to archive 2024-11-11T07:29:46,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:29:46,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/dd5b766af4f246b8a140d6ef38ac589b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/dd5b766af4f246b8a140d6ef38ac589b 2024-11-11T07:29:46,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/0c0583b3a8f3421dbc04b1f391213259 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/0c0583b3a8f3421dbc04b1f391213259 2024-11-11T07:29:46,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/347541acc3cb4c36bf671748aa341172 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/347541acc3cb4c36bf671748aa341172 2024-11-11T07:29:46,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/eb707fd6a6e6474faea26229db2d87f9 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/eb707fd6a6e6474faea26229db2d87f9 2024-11-11T07:29:46,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/56d8fd37ad974ae18218b2ecb6fa2ed5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/56d8fd37ad974ae18218b2ecb6fa2ed5 2024-11-11T07:29:46,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/e01afd75d7bd4433a72186b489374df7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/e01afd75d7bd4433a72186b489374df7 2024-11-11T07:29:46,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/58c1fd5a7f5b4a49aee087d179c91c9e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/58c1fd5a7f5b4a49aee087d179c91c9e 2024-11-11T07:29:46,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/2630135164e74bb19d74239b7bfffc34 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/2630135164e74bb19d74239b7bfffc34 2024-11-11T07:29:46,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/581701ef4029468d8fb3079cd1c32c61 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/581701ef4029468d8fb3079cd1c32c61 2024-11-11T07:29:46,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/57e2f6d5db3c4c43b989f3104736be97 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/57e2f6d5db3c4c43b989f3104736be97 2024-11-11T07:29:46,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/d557d71835e54db19cdced17395dda7a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/d557d71835e54db19cdced17395dda7a 2024-11-11T07:29:46,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/f4794c295d7f4e40a9dbe4258a117ac1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/f4794c295d7f4e40a9dbe4258a117ac1 2024-11-11T07:29:46,426 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/316011ad1d8c43f38fd390b7976d9b34 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/316011ad1d8c43f38fd390b7976d9b34 2024-11-11T07:29:46,428 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/7632bdb3c1fe418fbbca5abb6ef753b4 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/7632bdb3c1fe418fbbca5abb6ef753b4 2024-11-11T07:29:46,430 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/26def2790f794d4694191a84c32aa3e3 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/26def2790f794d4694191a84c32aa3e3 2024-11-11T07:29:46,433 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/0399fd0f411b4d0bbda6ae7532947c90 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/0399fd0f411b4d0bbda6ae7532947c90 2024-11-11T07:29:46,436 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/fe69164e57c64fb2a5a6c0c2c2f130d1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/fe69164e57c64fb2a5a6c0c2c2f130d1 2024-11-11T07:29:46,444 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/9b525c81b620403aab11dbd302791120 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/9b525c81b620403aab11dbd302791120 2024-11-11T07:29:46,450 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/c812fd10ce444f6bac0264ec2029eb4a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/c812fd10ce444f6bac0264ec2029eb4a 2024-11-11T07:29:46,460 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/371429fd1da64bc0b2507cd703f215ce to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/371429fd1da64bc0b2507cd703f215ce 2024-11-11T07:29:46,476 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/1ce997309eb84ea0b2f60e8824cfada5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/1ce997309eb84ea0b2f60e8824cfada5 2024-11-11T07:29:46,478 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/45cdd05e904b4bef9212b63f81f728dd to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/45cdd05e904b4bef9212b63f81f728dd 2024-11-11T07:29:46,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/186775dd7d9945de88feb0e8badbe8d0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/186775dd7d9945de88feb0e8badbe8d0 2024-11-11T07:29:46,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/61994602579047d4b484691b6ec67585 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/61994602579047d4b484691b6ec67585 2024-11-11T07:29:46,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/480b633b5e524c9aae4bcb715a0b424a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/480b633b5e524c9aae4bcb715a0b424a 2024-11-11T07:29:46,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/41d93c52e94f4d46ba390bfa6c2bc7e8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/41d93c52e94f4d46ba390bfa6c2bc7e8 2024-11-11T07:29:46,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/cc71e716ac064ad983523ca83c07ace5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/cc71e716ac064ad983523ca83c07ace5 2024-11-11T07:29:46,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/6fe60e9fd81b44b4995e872b4bd1381d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/6fe60e9fd81b44b4995e872b4bd1381d 2024-11-11T07:29:46,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/7880a086d9c445e68ec7da610567ee70 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/7880a086d9c445e68ec7da610567ee70 2024-11-11T07:29:46,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/ce392774c7ab4ecda1e8e43a871939d7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/ce392774c7ab4ecda1e8e43a871939d7 2024-11-11T07:29:46,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/d6f1ebdedb7947149c11bec2802c9dbc to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/d6f1ebdedb7947149c11bec2802c9dbc 2024-11-11T07:29:46,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/72b1f346241e442787f88ecb5d3bc002 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/72b1f346241e442787f88ecb5d3bc002 2024-11-11T07:29:46,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/3ac9c8726b3245b0ac8c81f6d5ed2a2e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/3ac9c8726b3245b0ac8c81f6d5ed2a2e 2024-11-11T07:29:46,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/6417a5ee06c949f894dea6ee0841d1c5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/6417a5ee06c949f894dea6ee0841d1c5 2024-11-11T07:29:46,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/4e79c6f1a9b845d18f0b1fe8c70d397c to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/4e79c6f1a9b845d18f0b1fe8c70d397c 2024-11-11T07:29:46,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/da72250a6b804265bc5b3f5b4b73587c to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/da72250a6b804265bc5b3f5b4b73587c 2024-11-11T07:29:46,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/c0e1dae6f0774568ab0eb28f8629613a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ff96e4e528fc42cea89f75e9496ed252, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ac63d3a643144c8bb260c745b9852812, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/c289bc3659984f9697d3adedd8b96fe4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/8833d1bd5c8243c1bfbdd50b590ff8b4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/2501523d99f9496b8d7b32b9c6893e65, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/68a694390b394396ac084056f2fbfbdd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/4dc0978f596d4cdfbc97943f653b6cfe, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/833824de1a074a7b9bb40f47acda6f41, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/02a6e22c648f41ed9a1b5f18faa62abf, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ae716c1fca3647da8e83189ed6167c22, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/df8d487b46844c39a05489f2994b941a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/68f01577b36e484db801284b81e11699, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/72a85c31adc84d298125bd9c50e710a0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/a91ea190016d486bbf94283104166420, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/d234de99693144ebb32bbdf269c5c174, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/f3de1cc47000450b9b93bb5898e1682b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/f62ba4ff0d37483a95ff0560ff5aeee2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/34155f8b9b14479e97c0a64c537d08ed, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/763e5c4582614d399726b1aff3b9a563, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/0aadbfd67bf2456fb7ef98b55754f4c3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/cc82b143f01c4d099bcf57b01d32ac54, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/6c2790a536b14ca6a0ccc630056690a2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/933bf28a59554c88a8afc43be9e4c16a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/6480886905c24839ac9df6c0ed76bf3e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/a56c2ef38a504ba59aaae3094a6be39a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/061a2e03d91f41d6a54c9abbe3926673, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/96e5e10bc5ae40b2a8f47c36eb42eeea, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/4e99151ca20846d69006b3f980e469b7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/0c7df2d9a34543aba2932ac8f67296b8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ddc3635c643c4a0da2912211b470273d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/cb79caa35f1d4ae49ec99c48f27b5e01, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/f6abdbece507441d82dde0b3d93d9d00, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/fa0725b1361745908f245f4a359db629, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/2dd6a29287b24903bf1d233e892c7b97, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/7bfe43dfd6b942e99993cd3e78f17499] to archive 2024-11-11T07:29:46,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:29:46,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/c0e1dae6f0774568ab0eb28f8629613a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/c0e1dae6f0774568ab0eb28f8629613a 2024-11-11T07:29:46,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ff96e4e528fc42cea89f75e9496ed252 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ff96e4e528fc42cea89f75e9496ed252 2024-11-11T07:29:46,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ac63d3a643144c8bb260c745b9852812 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ac63d3a643144c8bb260c745b9852812 2024-11-11T07:29:46,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/c289bc3659984f9697d3adedd8b96fe4 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/c289bc3659984f9697d3adedd8b96fe4 2024-11-11T07:29:46,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/8833d1bd5c8243c1bfbdd50b590ff8b4 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/8833d1bd5c8243c1bfbdd50b590ff8b4 2024-11-11T07:29:46,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/2501523d99f9496b8d7b32b9c6893e65 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/2501523d99f9496b8d7b32b9c6893e65 2024-11-11T07:29:46,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/68a694390b394396ac084056f2fbfbdd to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/68a694390b394396ac084056f2fbfbdd 2024-11-11T07:29:46,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/4dc0978f596d4cdfbc97943f653b6cfe to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/4dc0978f596d4cdfbc97943f653b6cfe 2024-11-11T07:29:46,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/833824de1a074a7b9bb40f47acda6f41 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/833824de1a074a7b9bb40f47acda6f41 2024-11-11T07:29:46,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/02a6e22c648f41ed9a1b5f18faa62abf to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/02a6e22c648f41ed9a1b5f18faa62abf 2024-11-11T07:29:46,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ae716c1fca3647da8e83189ed6167c22 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ae716c1fca3647da8e83189ed6167c22 2024-11-11T07:29:46,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/df8d487b46844c39a05489f2994b941a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/df8d487b46844c39a05489f2994b941a 2024-11-11T07:29:46,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/68f01577b36e484db801284b81e11699 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/68f01577b36e484db801284b81e11699 2024-11-11T07:29:46,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/72a85c31adc84d298125bd9c50e710a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/72a85c31adc84d298125bd9c50e710a0 2024-11-11T07:29:46,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/a91ea190016d486bbf94283104166420 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/a91ea190016d486bbf94283104166420 2024-11-11T07:29:46,597 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/d234de99693144ebb32bbdf269c5c174 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/d234de99693144ebb32bbdf269c5c174 2024-11-11T07:29:46,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/f3de1cc47000450b9b93bb5898e1682b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/f3de1cc47000450b9b93bb5898e1682b 2024-11-11T07:29:46,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/f62ba4ff0d37483a95ff0560ff5aeee2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/f62ba4ff0d37483a95ff0560ff5aeee2 2024-11-11T07:29:46,609 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/34155f8b9b14479e97c0a64c537d08ed to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/34155f8b9b14479e97c0a64c537d08ed 2024-11-11T07:29:46,616 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/763e5c4582614d399726b1aff3b9a563 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/763e5c4582614d399726b1aff3b9a563 2024-11-11T07:29:46,618 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/0aadbfd67bf2456fb7ef98b55754f4c3 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/0aadbfd67bf2456fb7ef98b55754f4c3 2024-11-11T07:29:46,619 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/cc82b143f01c4d099bcf57b01d32ac54 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/cc82b143f01c4d099bcf57b01d32ac54 2024-11-11T07:29:46,621 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/6c2790a536b14ca6a0ccc630056690a2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/6c2790a536b14ca6a0ccc630056690a2 2024-11-11T07:29:46,623 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/933bf28a59554c88a8afc43be9e4c16a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/933bf28a59554c88a8afc43be9e4c16a 2024-11-11T07:29:46,624 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/6480886905c24839ac9df6c0ed76bf3e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/6480886905c24839ac9df6c0ed76bf3e 2024-11-11T07:29:46,628 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/a56c2ef38a504ba59aaae3094a6be39a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/a56c2ef38a504ba59aaae3094a6be39a 2024-11-11T07:29:46,630 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/061a2e03d91f41d6a54c9abbe3926673 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/061a2e03d91f41d6a54c9abbe3926673 2024-11-11T07:29:46,631 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/96e5e10bc5ae40b2a8f47c36eb42eeea to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/96e5e10bc5ae40b2a8f47c36eb42eeea 2024-11-11T07:29:46,633 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/4e99151ca20846d69006b3f980e469b7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/4e99151ca20846d69006b3f980e469b7 2024-11-11T07:29:46,635 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/0c7df2d9a34543aba2932ac8f67296b8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/0c7df2d9a34543aba2932ac8f67296b8 2024-11-11T07:29:46,640 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ddc3635c643c4a0da2912211b470273d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ddc3635c643c4a0da2912211b470273d 2024-11-11T07:29:46,644 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/cb79caa35f1d4ae49ec99c48f27b5e01 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/cb79caa35f1d4ae49ec99c48f27b5e01 2024-11-11T07:29:46,646 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/f6abdbece507441d82dde0b3d93d9d00 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/f6abdbece507441d82dde0b3d93d9d00 2024-11-11T07:29:46,648 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/fa0725b1361745908f245f4a359db629 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/fa0725b1361745908f245f4a359db629 2024-11-11T07:29:46,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/2dd6a29287b24903bf1d233e892c7b97 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/2dd6a29287b24903bf1d233e892c7b97 2024-11-11T07:29:46,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/7bfe43dfd6b942e99993cd3e78f17499 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/7bfe43dfd6b942e99993cd3e78f17499 2024-11-11T07:29:46,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/5f6d1b1486b148a184f1f7ffb67fe132, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/956eae2b6a1a41608cec60f188170af8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/8166a7110f0f4a16a99152ea0fa1d27c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/442c137e72a4432c874cbcb9cbaccda8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/09ed1eb053a64ab4b49bdfb2370686e6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/665866aabf8243fa8d7223919a804c31, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/2fccaa737f0f4fbc9dcee45b420d0116, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6ddf9490083b493f91ac6c7e7f92e8b6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/3d00ff896395484bb2f0557b631947b1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/c891743d72dd44179db7f74b444550c0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/09a43f92441f4d7583566a2be0e00d64, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/97295379671d4834b809216f4d6f9958, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/df102dc92683480dae118dacad15df46, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/a0c1a08c5f1046f6812adfcc6ea3c633, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/1c9823913b1b4021a2f1a294bbf1136d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/46f2faec2361461b84a08987806b0091, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/b790366eebf747a0b3c4c01f403e0d59, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/89e8a61365a74b108a2dbb0dde1e9431, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/c81f07e158214bf981190161583947a7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/ad1364e1c72048c79fb62185970fe194, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/d3bcd016fcb74b31b312aae6840bbc46, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/9dcb094dbd71427f8dfce24285e3cded, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/f54669a8291548f1a16ee5d32f4e7962, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6fd4af622be44a8f95e386c8f56790b6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/d3ce650b69e94591b45b61cab65f5483, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6753ae2537b1434584eaa75afabbfef7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/9a15f3a6f4204d99bc36826cf9e68703, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/18f3ad3092a041dc96534005a4a90a89, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/8f83bb02104e4d1bb922c655f25bd843, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/acac6f07a400443d81bd525bdde833f6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/aa34c444e3594db68fc698034c48cf97, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/2f5c39dcc84f4c15841773f5946d10d3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/06934fc46b62464f840f7ea38c9d507d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/b559002437844fb483c09fd0ba265ce3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/f3c9153db89e4c969f8272de7c0e67f9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/90c6b73f98584401a5707d6d950a1477] to archive 2024-11-11T07:29:46,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:29:46,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/5f6d1b1486b148a184f1f7ffb67fe132 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/5f6d1b1486b148a184f1f7ffb67fe132 2024-11-11T07:29:46,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/956eae2b6a1a41608cec60f188170af8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/956eae2b6a1a41608cec60f188170af8 2024-11-11T07:29:46,683 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/8166a7110f0f4a16a99152ea0fa1d27c to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/8166a7110f0f4a16a99152ea0fa1d27c 2024-11-11T07:29:46,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/442c137e72a4432c874cbcb9cbaccda8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/442c137e72a4432c874cbcb9cbaccda8 2024-11-11T07:29:46,691 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/09ed1eb053a64ab4b49bdfb2370686e6 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/09ed1eb053a64ab4b49bdfb2370686e6 2024-11-11T07:29:46,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/665866aabf8243fa8d7223919a804c31 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/665866aabf8243fa8d7223919a804c31 2024-11-11T07:29:46,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/2fccaa737f0f4fbc9dcee45b420d0116 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/2fccaa737f0f4fbc9dcee45b420d0116 2024-11-11T07:29:46,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6ddf9490083b493f91ac6c7e7f92e8b6 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6ddf9490083b493f91ac6c7e7f92e8b6 2024-11-11T07:29:46,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/3d00ff896395484bb2f0557b631947b1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/3d00ff896395484bb2f0557b631947b1 2024-11-11T07:29:46,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/c891743d72dd44179db7f74b444550c0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/c891743d72dd44179db7f74b444550c0 2024-11-11T07:29:46,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/09a43f92441f4d7583566a2be0e00d64 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/09a43f92441f4d7583566a2be0e00d64 2024-11-11T07:29:46,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/97295379671d4834b809216f4d6f9958 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/97295379671d4834b809216f4d6f9958 2024-11-11T07:29:46,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/df102dc92683480dae118dacad15df46 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/df102dc92683480dae118dacad15df46 2024-11-11T07:29:46,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/a0c1a08c5f1046f6812adfcc6ea3c633 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/a0c1a08c5f1046f6812adfcc6ea3c633 2024-11-11T07:29:46,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/1c9823913b1b4021a2f1a294bbf1136d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/1c9823913b1b4021a2f1a294bbf1136d 2024-11-11T07:29:46,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/46f2faec2361461b84a08987806b0091 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/46f2faec2361461b84a08987806b0091 2024-11-11T07:29:46,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/b790366eebf747a0b3c4c01f403e0d59 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/b790366eebf747a0b3c4c01f403e0d59 2024-11-11T07:29:46,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/89e8a61365a74b108a2dbb0dde1e9431 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/89e8a61365a74b108a2dbb0dde1e9431 2024-11-11T07:29:46,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/c81f07e158214bf981190161583947a7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/c81f07e158214bf981190161583947a7 2024-11-11T07:29:46,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/ad1364e1c72048c79fb62185970fe194 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/ad1364e1c72048c79fb62185970fe194 2024-11-11T07:29:46,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/d3bcd016fcb74b31b312aae6840bbc46 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/d3bcd016fcb74b31b312aae6840bbc46 2024-11-11T07:29:46,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/9dcb094dbd71427f8dfce24285e3cded to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/9dcb094dbd71427f8dfce24285e3cded 2024-11-11T07:29:46,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/f54669a8291548f1a16ee5d32f4e7962 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/f54669a8291548f1a16ee5d32f4e7962 2024-11-11T07:29:46,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6fd4af622be44a8f95e386c8f56790b6 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6fd4af622be44a8f95e386c8f56790b6 2024-11-11T07:29:46,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/d3ce650b69e94591b45b61cab65f5483 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/d3ce650b69e94591b45b61cab65f5483 2024-11-11T07:29:46,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6753ae2537b1434584eaa75afabbfef7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/6753ae2537b1434584eaa75afabbfef7 2024-11-11T07:29:46,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/9a15f3a6f4204d99bc36826cf9e68703 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/9a15f3a6f4204d99bc36826cf9e68703 2024-11-11T07:29:46,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/18f3ad3092a041dc96534005a4a90a89 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/18f3ad3092a041dc96534005a4a90a89 2024-11-11T07:29:46,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/8f83bb02104e4d1bb922c655f25bd843 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/8f83bb02104e4d1bb922c655f25bd843 2024-11-11T07:29:46,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/acac6f07a400443d81bd525bdde833f6 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/acac6f07a400443d81bd525bdde833f6 2024-11-11T07:29:46,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/aa34c444e3594db68fc698034c48cf97 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/aa34c444e3594db68fc698034c48cf97 2024-11-11T07:29:46,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/2f5c39dcc84f4c15841773f5946d10d3 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/2f5c39dcc84f4c15841773f5946d10d3 2024-11-11T07:29:46,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/06934fc46b62464f840f7ea38c9d507d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/06934fc46b62464f840f7ea38c9d507d 2024-11-11T07:29:46,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/b559002437844fb483c09fd0ba265ce3 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/b559002437844fb483c09fd0ba265ce3 2024-11-11T07:29:46,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/f3c9153db89e4c969f8272de7c0e67f9 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/f3c9153db89e4c969f8272de7c0e67f9 2024-11-11T07:29:46,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/90c6b73f98584401a5707d6d950a1477 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/90c6b73f98584401a5707d6d950a1477 2024-11-11T07:29:46,826 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/recovered.edits/620.seqid, newMaxSeqId=620, maxSeqId=1 2024-11-11T07:29:46,833 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. 2024-11-11T07:29:46,834 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 6756dec3886a521f131ee738c3465dc1: Waiting for close lock at 1731310185045Running coprocessor pre-close hooks at 1731310185046 (+1 ms)Disabling compacts and flushes for region at 1731310185046Disabling writes for close at 1731310185046Obtaining lock to block concurrent updates at 1731310185046Preparing flush snapshotting stores in 6756dec3886a521f131ee738c3465dc1 at 1731310185046Finished memstore snapshotting TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731310185047 (+1 ms)Flushing stores of TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1. at 1731310185047Flushing 6756dec3886a521f131ee738c3465dc1/A: creating writer at 1731310185047Flushing 6756dec3886a521f131ee738c3465dc1/A: appending metadata at 1731310185050 (+3 ms)Flushing 6756dec3886a521f131ee738c3465dc1/A: closing flushed file at 1731310185050Flushing 6756dec3886a521f131ee738c3465dc1/B: creating writer at 1731310185461 (+411 ms)Flushing 6756dec3886a521f131ee738c3465dc1/B: appending metadata at 1731310185465 (+4 ms)Flushing 6756dec3886a521f131ee738c3465dc1/B: closing flushed file at 1731310185465Flushing 6756dec3886a521f131ee738c3465dc1/C: creating writer at 1731310185879 (+414 ms)Flushing 6756dec3886a521f131ee738c3465dc1/C: appending metadata at 1731310185882 (+3 ms)Flushing 6756dec3886a521f131ee738c3465dc1/C: closing flushed file at 1731310185882Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@129778fa: reopening flushed file at 1731310186317 (+435 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@315b5cee: reopening flushed file at 1731310186327 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d67a7d9: reopening flushed file at 1731310186336 (+9 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 6756dec3886a521f131ee738c3465dc1 in 1321ms, sequenceid=617, compaction requested=false at 1731310186367 (+31 ms)Writing region close event to WAL at 1731310186794 (+427 ms)Running coprocessor post-close hooks at 1731310186833 (+39 ms)Closed at 1731310186833 2024-11-11T07:29:46,851 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:46,868 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=6756dec3886a521f131ee738c3465dc1, regionState=CLOSED 2024-11-11T07:29:46,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:29:46,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-11T07:29:46,890 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 6756dec3886a521f131ee738c3465dc1, server=ce2cf0244921,33345,1731310155369 in 1.9990 sec 2024-11-11T07:29:46,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-11T07:29:46,897 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.0280 sec 2024-11-11T07:29:46,900 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310186899"}]},"ts":"1731310186899"} 2024-11-11T07:29:46,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-11T07:29:46,900 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6756dec3886a521f131ee738c3465dc1, UNASSIGN in 2.0170 sec 2024-11-11T07:29:46,904 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T07:29:46,904 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T07:29:46,907 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.0510 sec 2024-11-11T07:29:47,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-11T07:29:47,005 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:47,009 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T07:29:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:29:47,016 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:29:47,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T07:29:47,018 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:29:47,024 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:47,032 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/recovered.edits] 2024-11-11T07:29:47,037 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/3baaf1be5575412f8b7ba2ccdea80064 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/3baaf1be5575412f8b7ba2ccdea80064 2024-11-11T07:29:47,038 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/7bca83ac2efd4a6ba519a6d42e3c0606 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/A/7bca83ac2efd4a6ba519a6d42e3c0606 2024-11-11T07:29:47,041 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/64097deb5c2e473aabfce3f9c1e6957a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/64097deb5c2e473aabfce3f9c1e6957a 2024-11-11T07:29:47,044 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ba7be5fdc9a749abac5def58c574980d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/B/ba7be5fdc9a749abac5def58c574980d 2024-11-11T07:29:47,050 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/e4170eba32304ce09c3470570220b850 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/e4170eba32304ce09c3470570220b850 2024-11-11T07:29:47,052 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/f2aa2be27f2a4b6489e24f76054a2f77 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/C/f2aa2be27f2a4b6489e24f76054a2f77 2024-11-11T07:29:47,057 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/recovered.edits/620.seqid to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1/recovered.edits/620.seqid 2024-11-11T07:29:47,058 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6756dec3886a521f131ee738c3465dc1 2024-11-11T07:29:47,058 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T07:29:47,068 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:29:47,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-11T07:29:47,081 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T07:29:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T07:29:47,151 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T07:29:47,154 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:29:47,154 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T07:29:47,155 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731310187154"}]},"ts":"9223372036854775807"} 2024-11-11T07:29:47,160 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T07:29:47,160 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6756dec3886a521f131ee738c3465dc1, NAME => 'TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T07:29:47,161 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T07:29:47,161 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731310187161"}]},"ts":"9223372036854775807"} 2024-11-11T07:29:47,165 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T07:29:47,168 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:29:47,171 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 158 msec 2024-11-11T07:29:47,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-11T07:29:47,335 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T07:29:47,336 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:47,350 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=248 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_503664647_22 at /127.0.0.1:44774 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_606034174_22 at /127.0.0.1:39882 [Waiting for operation #236] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;ce2cf0244921:33345-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=838 (was 750) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3257 (was 3211) - AvailableMemoryMB LEAK? - 2024-11-11T07:29:47,360 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=838, ProcessCount=11, AvailableMemoryMB=3257 2024-11-11T07:29:47,362 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:29:47,362 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:29:47,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T07:29:47,364 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T07:29:47,365 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:47,365 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-11T07:29:47,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T07:29:47,366 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T07:29:47,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741951_1127 (size=1061) 2024-11-11T07:29:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T07:29:47,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T07:29:47,774 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e 2024-11-11T07:29:47,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741952_1128 (size=53) 2024-11-11T07:29:47,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T07:29:48,187 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:29:48,187 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 18312f0c839a758e8897bb39c81d93ff, disabling compactions & flushes 2024-11-11T07:29:48,187 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:48,187 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:48,187 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. after waiting 0 ms 2024-11-11T07:29:48,187 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:48,187 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:48,187 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 18312f0c839a758e8897bb39c81d93ff: Waiting for close lock at 1731310188187Disabling compacts and flushes for region at 1731310188187Disabling writes for close at 1731310188187Writing region close event to WAL at 1731310188187Closed at 1731310188187 2024-11-11T07:29:48,188 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T07:29:48,189 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731310188188"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731310188188"}]},"ts":"1731310188188"} 2024-11-11T07:29:48,191 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T07:29:48,192 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T07:29:48,192 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310188192"}]},"ts":"1731310188192"} 2024-11-11T07:29:48,194 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T07:29:48,194 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18312f0c839a758e8897bb39c81d93ff, ASSIGN}] 2024-11-11T07:29:48,196 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18312f0c839a758e8897bb39c81d93ff, ASSIGN 2024-11-11T07:29:48,196 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=18312f0c839a758e8897bb39c81d93ff, ASSIGN; state=OFFLINE, location=ce2cf0244921,33345,1731310155369; forceNewPlan=false, retain=false 2024-11-11T07:29:48,348 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=18312f0c839a758e8897bb39c81d93ff, regionState=OPENING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:48,350 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=18312f0c839a758e8897bb39c81d93ff, ASSIGN because future has completed 2024-11-11T07:29:48,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:29:48,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T07:29:48,506 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:48,506 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:29:48,507 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:48,507 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:29:48,507 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:48,507 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:48,508 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:48,509 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:29:48,510 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18312f0c839a758e8897bb39c81d93ff columnFamilyName A 2024-11-11T07:29:48,511 DEBUG [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:48,511 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.HStore(327): Store=18312f0c839a758e8897bb39c81d93ff/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:29:48,511 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:48,513 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:29:48,513 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18312f0c839a758e8897bb39c81d93ff columnFamilyName B 2024-11-11T07:29:48,513 DEBUG [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:48,514 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.HStore(327): Store=18312f0c839a758e8897bb39c81d93ff/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:29:48,514 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:48,515 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:29:48,515 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18312f0c839a758e8897bb39c81d93ff columnFamilyName C 2024-11-11T07:29:48,515 DEBUG [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:48,515 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.HStore(327): Store=18312f0c839a758e8897bb39c81d93ff/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:29:48,516 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:48,516 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:48,516 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:48,517 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:48,518 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:48,518 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:48,518 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:29:48,519 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:48,521 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T07:29:48,522 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 18312f0c839a758e8897bb39c81d93ff; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72627932, jitterRate=0.0822405219078064}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:29:48,522 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:48,523 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 18312f0c839a758e8897bb39c81d93ff: Running coprocessor pre-open hook at 1731310188507Writing region info on filesystem at 1731310188507Initializing all the Stores at 1731310188508 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310188508Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310188508Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310188508Cleaning up temporary data from old regions at 1731310188518 (+10 ms)Running coprocessor post-open hooks at 1731310188522 (+4 ms)Region opened successfully at 1731310188523 (+1 ms) 2024-11-11T07:29:48,524 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., pid=34, masterSystemTime=1731310188502 2024-11-11T07:29:48,528 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:48,528 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:48,528 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=18312f0c839a758e8897bb39c81d93ff, regionState=OPEN, openSeqNum=2, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:48,530 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:29:48,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-11T07:29:48,534 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 in 181 msec 2024-11-11T07:29:48,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-11T07:29:48,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18312f0c839a758e8897bb39c81d93ff, ASSIGN in 340 msec 2024-11-11T07:29:48,537 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T07:29:48,537 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310188537"}]},"ts":"1731310188537"} 2024-11-11T07:29:48,539 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T07:29:48,540 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T07:29:48,542 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-11-11T07:29:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-11T07:29:49,515 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:49,516 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2697ccde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:49,516 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:49,516 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:49,518 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e528e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:49,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:49,519 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:49,520 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32792, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:49,520 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53186973, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:49,521 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:49,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:49,522 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:49,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:49,525 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T07:29:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is ce2cf0244921,33029,1731310154427 2024-11-11T07:29:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@18303f2f 2024-11-11T07:29:49,526 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T07:29:49,527 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T07:29:49,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:29:49,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:29:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T07:29:49,553 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741953_1129 (size=1097) 2024-11-11T07:29:49,955 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T07:29:49,955 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T07:29:49,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:29:49,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18312f0c839a758e8897bb39c81d93ff, REOPEN/MOVE}] 2024-11-11T07:29:49,968 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18312f0c839a758e8897bb39c81d93ff, REOPEN/MOVE 2024-11-11T07:29:49,969 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=18312f0c839a758e8897bb39c81d93ff, regionState=CLOSING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:49,970 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=18312f0c839a758e8897bb39c81d93ff, REOPEN/MOVE because future has completed 2024-11-11T07:29:49,971 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:29:49,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:29:50,123 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,123 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:29:50,123 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 18312f0c839a758e8897bb39c81d93ff, disabling compactions & flushes 2024-11-11T07:29:50,123 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:50,123 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:50,123 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. after waiting 0 ms 2024-11-11T07:29:50,123 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:50,127 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T07:29:50,128 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:50,128 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 18312f0c839a758e8897bb39c81d93ff: Waiting for close lock at 1731310190123Running coprocessor pre-close hooks at 1731310190123Disabling compacts and flushes for region at 1731310190123Disabling writes for close at 1731310190123Writing region close event to WAL at 1731310190124 (+1 ms)Running coprocessor post-close hooks at 1731310190127 (+3 ms)Closed at 1731310190127 2024-11-11T07:29:50,128 WARN [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 18312f0c839a758e8897bb39c81d93ff to self. 2024-11-11T07:29:50,130 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,131 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=18312f0c839a758e8897bb39c81d93ff, regionState=CLOSED 2024-11-11T07:29:50,132 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:29:50,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-11T07:29:50,135 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 in 162 msec 2024-11-11T07:29:50,136 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=18312f0c839a758e8897bb39c81d93ff, REOPEN/MOVE; state=CLOSED, location=ce2cf0244921,33345,1731310155369; forceNewPlan=false, retain=true 2024-11-11T07:29:50,287 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=18312f0c839a758e8897bb39c81d93ff, regionState=OPENING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:50,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=18312f0c839a758e8897bb39c81d93ff, REOPEN/MOVE because future has completed 2024-11-11T07:29:50,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:29:50,449 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:50,449 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:29:50,450 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,450 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:29:50,450 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,450 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,455 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,456 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:29:50,461 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18312f0c839a758e8897bb39c81d93ff columnFamilyName A 2024-11-11T07:29:50,463 DEBUG [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:50,464 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.HStore(327): Store=18312f0c839a758e8897bb39c81d93ff/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:29:50,464 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,465 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:29:50,465 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18312f0c839a758e8897bb39c81d93ff columnFamilyName B 2024-11-11T07:29:50,465 DEBUG [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:50,466 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.HStore(327): Store=18312f0c839a758e8897bb39c81d93ff/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:29:50,466 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,467 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:29:50,467 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 18312f0c839a758e8897bb39c81d93ff columnFamilyName C 2024-11-11T07:29:50,467 DEBUG [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:50,468 INFO [StoreOpener-18312f0c839a758e8897bb39c81d93ff-1 {}] regionserver.HStore(327): Store=18312f0c839a758e8897bb39c81d93ff/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:29:50,468 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:50,468 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,469 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,469 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,470 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,470 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,471 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:29:50,472 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,473 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 18312f0c839a758e8897bb39c81d93ff; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61876602, jitterRate=-0.07796677947044373}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:29:50,473 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,476 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 18312f0c839a758e8897bb39c81d93ff: Running coprocessor pre-open hook at 1731310190450Writing region info on filesystem at 1731310190450Initializing all the Stores at 1731310190451 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310190451Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310190451Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310190451Cleaning up temporary data from old regions at 1731310190470 (+19 ms)Running coprocessor post-open hooks at 1731310190473 (+3 ms)Region opened successfully at 1731310190476 (+3 ms) 2024-11-11T07:29:50,477 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., pid=39, masterSystemTime=1731310190445 2024-11-11T07:29:50,479 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:50,480 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:50,480 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=18312f0c839a758e8897bb39c81d93ff, regionState=OPEN, openSeqNum=5, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:50,482 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:29:50,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-11T07:29:50,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 in 194 msec 2024-11-11T07:29:50,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-11T07:29:50,488 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18312f0c839a758e8897bb39c81d93ff, REOPEN/MOVE in 519 msec 2024-11-11T07:29:50,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-11T07:29:50,493 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 531 msec 2024-11-11T07:29:50,497 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 958 msec 2024-11-11T07:29:50,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-11T07:29:50,500 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:50,500 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cb008, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:50,501 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:50,502 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:50,502 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:50,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a7e74e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:50,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:50,503 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:50,504 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32822, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:50,505 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7eb2646, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35409f72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,505 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:50,506 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:50,507 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55edbc69, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:50,507 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:50,508 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:50,509 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32842, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:50,510 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@764155aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,512 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7944f9cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,512 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:50,512 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:50,513 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:50,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:50,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@173cf686, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:50,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:50,515 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:50,516 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b45b03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38720fa6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,517 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:50,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:50,518 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:50,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f99951e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:50,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:50,520 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32880, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:50,521 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40b1e54b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,521 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c77b203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,522 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:50,522 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:50,523 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:50,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:50,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:50,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79f7d2d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:50,523 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:50,524 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:50,525 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32900, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:50,526 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@303b49a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,531 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e3f6eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,531 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:50,531 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:50,532 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4684d61e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,532 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:50,533 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:50,534 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32912, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:50,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b15666b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,535 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40353459, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,535 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:50,536 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:50,536 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:50,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:50,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:50,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a39d9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:50,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:50,537 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:50,539 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:50,539 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c22aa02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,540 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@160383af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,540 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:50,540 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:50,541 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:50,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:50,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:50,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6874eb42, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,541 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:50,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:50,542 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:50,543 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:50,544 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47775e8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,544 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123a694c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,544 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:29:50,545 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:29:50,545 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:29:50,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:29:50,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:29:50,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d52dfbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:29:50,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:29:50,546 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:29:50,547 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:32984, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:29:50,548 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77fb3b87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:29:50,549 DEBUG [Thread-652 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:50,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:50,551 DEBUG [Thread-665 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:50,552 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60806, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:50,552 DEBUG [Thread-656 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:50,553 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:29:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:50,555 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:50,556 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60814, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:50,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-11T07:29:50,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T07:29:50,557 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:29:50,558 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:50,559 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:29:50,559 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:29:50,561 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:29:50,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:29:50,566 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:29:50,571 DEBUG [Thread-659 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:50,571 DEBUG [Thread-661 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:50,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:50,576 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60826, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:50,584 DEBUG [Thread-661 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:50,585 DEBUG [Thread-661 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:50,587 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:50,587 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:50,588 DEBUG [Thread-648 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:50,588 DEBUG [Thread-648 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:50,590 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:50,590 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:29:50,592 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:29:50,595 DEBUG [Thread-654 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:29:50,596 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:50,599 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60860, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:50,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T07:29:50,603 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:29:50,603 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:29:50,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:50,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:29:50,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:50,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:29:50,604 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:50,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,611 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:50,618 DEBUG [Thread-650 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:50,619 DEBUG [Thread-650 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:50,620 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:50,627 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:29:50,655 DEBUG [Thread-663 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:29:50,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:29:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:29:50,659 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60876, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:29:50,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:29:50,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T07:29:50,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731310190642, max timestamp=1731310190665Num uniques 859; 2024-11-11T07:29:50,677 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T07:29:50,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731310190642, max timestamp=1731310190665Num uniques 859; 2024-11-11T07:29:50,680 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T07:29:50,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731310190642, max timestamp=1731310190665Num uniques 859; 2024-11-11T07:29:50,683 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T07:29:50,715 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T07:29:50,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:50,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:29:50,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:50,716 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:50,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:50,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:50,732 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116308b506564b4381b1e9f7a64214eff8_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310190586/Put/seqid=0 2024-11-11T07:29:50,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:50,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:50,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310250747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:50,775 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:50,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310250749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,776 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,779 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:50,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310250744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:50,780 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:50,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741954_1130 (size=19474) 2024-11-11T07:29:50,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310250752, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,780 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:50,781 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:50,781 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:50,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310250753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:50,788 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116308b506564b4381b1e9f7a64214eff8_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116308b506564b4381b1e9f7a64214eff8_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:50,789 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/82d8c2e6825b414d93b6edd0c287638d, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:29:50,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/82d8c2e6825b414d93b6edd0c287638d is 175, key is test_row_0/A:col10/1731310190586/Put/seqid=0 2024-11-11T07:29:50,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741955_1131 (size=56733) 2024-11-11T07:29:50,837 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/82d8c2e6825b414d93b6edd0c287638d 2024-11-11T07:29:50,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T07:29:50,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T07:29:50,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:50,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:29:50,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:50,877 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:50,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:50,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/4fcfcaef13f24c4098e67b2fa9988245 is 50, key is test_row_0/B:col10/1731310190586/Put/seqid=0 2024-11-11T07:29:50,895 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:50,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310250885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:50,896 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:50,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310250885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,897 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310250885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:50,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310250887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,898 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:50,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:50,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310250903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,908 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:50,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:50,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741956_1132 (size=12001) 2024-11-11T07:29:50,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/4fcfcaef13f24c4098e67b2fa9988245 2024-11-11T07:29:51,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/9104fce7b44b463ab9f1c730173748ee is 50, key is test_row_0/C:col10/1731310190586/Put/seqid=0 2024-11-11T07:29:51,035 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T07:29:51,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:51,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:29:51,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:51,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:51,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:51,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:51,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741957_1133 (size=12001) 2024-11-11T07:29:51,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310251107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:51,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,110 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:51,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310251108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:51,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310251108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:51,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310251108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:51,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310251115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:51,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,121 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:51,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T07:29:51,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T07:29:51,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:51,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:29:51,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:51,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:51,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:51,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:51,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T07:29:51,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:51,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:29:51,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:51,361 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:51,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:51,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:51,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:51,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310251414, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,423 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:51,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:51,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310251415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:51,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:51,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310251415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:51,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:51,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310251425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,427 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:51,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:51,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310251425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:51,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:51,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:51,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/9104fce7b44b463ab9f1c730173748ee 2024-11-11T07:29:51,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/82d8c2e6825b414d93b6edd0c287638d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/82d8c2e6825b414d93b6edd0c287638d 2024-11-11T07:29:51,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/82d8c2e6825b414d93b6edd0c287638d, entries=300, sequenceid=16, filesize=55.4 K 2024-11-11T07:29:51,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/4fcfcaef13f24c4098e67b2fa9988245 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/4fcfcaef13f24c4098e67b2fa9988245 2024-11-11T07:29:51,515 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T07:29:51,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:51,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:29:51,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:51,516 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:51,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:51,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:51,527 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/4fcfcaef13f24c4098e67b2fa9988245, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T07:29:51,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/9104fce7b44b463ab9f1c730173748ee as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/9104fce7b44b463ab9f1c730173748ee 2024-11-11T07:29:51,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/9104fce7b44b463ab9f1c730173748ee, entries=150, sequenceid=16, filesize=11.7 K 2024-11-11T07:29:51,592 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=127.47 KB/130530 for 18312f0c839a758e8897bb39c81d93ff in 989ms, sequenceid=16, compaction requested=false 2024-11-11T07:29:51,592 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T07:29:51,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:29:51,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-11T07:29:51,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:51,691 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-11T07:29:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:29:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:29:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:29:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:29:51,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T07:29:51,759 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111aa21bf43ef0b4330a748429ba9638d45_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310190749/Put/seqid=0 2024-11-11T07:29:51,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741958_1134 (size=12154) 2024-11-11T07:29:51,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:51,829 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111aa21bf43ef0b4330a748429ba9638d45_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111aa21bf43ef0b4330a748429ba9638d45_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:51,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/1ebd3ac1149a4e13887f440e2ae79e92, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:29:51,838 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/1ebd3ac1149a4e13887f440e2ae79e92 is 175, key is test_row_0/A:col10/1731310190749/Put/seqid=0 2024-11-11T07:29:51,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741959_1135 (size=30955) 2024-11-11T07:29:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:51,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:29:51,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310251962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:51,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310251975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310251984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310251985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310251987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,003 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310252105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310252106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310252115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310252115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,118 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310252115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,123 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,298 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/1ebd3ac1149a4e13887f440e2ae79e92 2024-11-11T07:29:52,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310252319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,326 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,329 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310252324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310252326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310252326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310252325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,339 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/c476f1d305f54245a21e6fcb2ad2ae06 is 50, key is test_row_0/B:col10/1731310190749/Put/seqid=0 2024-11-11T07:29:52,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741960_1136 (size=12001) 2024-11-11T07:29:52,448 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T07:29:52,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310252634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310252647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,650 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310252648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,651 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310252648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,651 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:52,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310252650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:52,652 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:52,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T07:29:52,804 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/c476f1d305f54245a21e6fcb2ad2ae06 2024-11-11T07:29:52,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/692a372fa9d1456da7db589eb99874c4 is 50, key is test_row_0/C:col10/1731310190749/Put/seqid=0 2024-11-11T07:29:52,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741961_1137 (size=12001) 2024-11-11T07:29:53,156 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:53,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310253155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:53,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310253155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:53,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:53,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:53,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310253155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:53,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:53,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:53,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310253156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:53,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:53,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:53,160 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:53,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:53,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310253164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:53,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:53,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:53,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:53,243 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/692a372fa9d1456da7db589eb99874c4 2024-11-11T07:29:53,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/1ebd3ac1149a4e13887f440e2ae79e92 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/1ebd3ac1149a4e13887f440e2ae79e92 2024-11-11T07:29:53,255 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/1ebd3ac1149a4e13887f440e2ae79e92, entries=150, sequenceid=51, filesize=30.2 K 2024-11-11T07:29:53,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/c476f1d305f54245a21e6fcb2ad2ae06 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/c476f1d305f54245a21e6fcb2ad2ae06 2024-11-11T07:29:53,267 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/c476f1d305f54245a21e6fcb2ad2ae06, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T07:29:53,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/692a372fa9d1456da7db589eb99874c4 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/692a372fa9d1456da7db589eb99874c4 2024-11-11T07:29:53,276 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/692a372fa9d1456da7db589eb99874c4, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T07:29:53,279 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 18312f0c839a758e8897bb39c81d93ff in 1588ms, sequenceid=51, compaction requested=false 2024-11-11T07:29:53,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:29:53,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:53,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-11T07:29:53,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-11T07:29:53,283 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-11T07:29:53,283 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7220 sec 2024-11-11T07:29:53,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 2.7310 sec 2024-11-11T07:29:54,174 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T07:29:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:29:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:29:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:29:54,174 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:54,205 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117df7957dc3be4c568cd8d0699cb6c753_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310191973/Put/seqid=0 2024-11-11T07:29:54,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741962_1138 (size=14594) 2024-11-11T07:29:54,255 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310254238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,256 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,256 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310254240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310254248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310254251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,265 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310254256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,272 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310254367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,374 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310254367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,377 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,383 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310254379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,385 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310254379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,386 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310254379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,386 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,580 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310254575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,583 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,584 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310254585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,601 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310254594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310254594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,602 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310254595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,603 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,604 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,662 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:54,671 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117df7957dc3be4c568cd8d0699cb6c753_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117df7957dc3be4c568cd8d0699cb6c753_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:54,675 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/f9d9421655cc446f920930dc2adb32cf, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:29:54,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/f9d9421655cc446f920930dc2adb32cf is 175, key is test_row_0/A:col10/1731310191973/Put/seqid=0 2024-11-11T07:29:54,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741963_1139 (size=39549) 2024-11-11T07:29:54,714 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/f9d9421655cc446f920930dc2adb32cf 2024-11-11T07:29:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-11T07:29:54,737 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:54,740 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:29:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-11T07:29:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T07:29:54,744 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:29:54,745 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:29:54,745 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:29:54,748 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/3d3cfd10d43d4010905550500b6f04db is 50, key is test_row_0/B:col10/1731310191973/Put/seqid=0 2024-11-11T07:29:54,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741964_1140 (size=12001) 2024-11-11T07:29:54,809 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/3d3cfd10d43d4010905550500b6f04db 2024-11-11T07:29:54,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/1cdb0eead393426ca43ca26c402923e4 is 50, key is test_row_0/C:col10/1731310191973/Put/seqid=0 2024-11-11T07:29:54,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T07:29:54,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741965_1141 (size=12001) 2024-11-11T07:29:54,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T07:29:54,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:54,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:29:54,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:54,902 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:54,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:54,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310254895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310254895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,905 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:54,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310254904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,909 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,917 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310254915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310254915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:54,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:55,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T07:29:55,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T07:29:55,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:55,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:29:55,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:55,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:55,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:55,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:55,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T07:29:55,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:55,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:29:55,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:55,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:55,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:55,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:55,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/1cdb0eead393426ca43ca26c402923e4 2024-11-11T07:29:55,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/f9d9421655cc446f920930dc2adb32cf as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/f9d9421655cc446f920930dc2adb32cf 2024-11-11T07:29:55,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/f9d9421655cc446f920930dc2adb32cf, entries=200, sequenceid=63, filesize=38.6 K 2024-11-11T07:29:55,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/3d3cfd10d43d4010905550500b6f04db as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/3d3cfd10d43d4010905550500b6f04db 2024-11-11T07:29:55,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/3d3cfd10d43d4010905550500b6f04db, entries=150, sequenceid=63, filesize=11.7 K 2024-11-11T07:29:55,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/1cdb0eead393426ca43ca26c402923e4 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/1cdb0eead393426ca43ca26c402923e4 2024-11-11T07:29:55,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/1cdb0eead393426ca43ca26c402923e4, entries=150, sequenceid=63, filesize=11.7 K 2024-11-11T07:29:55,335 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 18312f0c839a758e8897bb39c81d93ff in 1162ms, sequenceid=63, compaction requested=true 2024-11-11T07:29:55,335 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:29:55,335 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:55,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:29:55,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:55,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:29:55,336 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:55,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:55,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:29:55,336 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:55,337 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127237 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:55,337 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:55,337 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/A is initiating minor compaction (all files) 2024-11-11T07:29:55,337 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/B is initiating minor compaction (all files) 2024-11-11T07:29:55,337 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/A in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:55,337 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/B in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:55,337 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/4fcfcaef13f24c4098e67b2fa9988245, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/c476f1d305f54245a21e6fcb2ad2ae06, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/3d3cfd10d43d4010905550500b6f04db] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=35.2 K 2024-11-11T07:29:55,337 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/82d8c2e6825b414d93b6edd0c287638d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/1ebd3ac1149a4e13887f440e2ae79e92, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/f9d9421655cc446f920930dc2adb32cf] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=124.3 K 2024-11-11T07:29:55,337 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:55,337 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/82d8c2e6825b414d93b6edd0c287638d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/1ebd3ac1149a4e13887f440e2ae79e92, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/f9d9421655cc446f920930dc2adb32cf] 2024-11-11T07:29:55,338 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fcfcaef13f24c4098e67b2fa9988245, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731310190584 2024-11-11T07:29:55,338 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting c476f1d305f54245a21e6fcb2ad2ae06, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731310190730 2024-11-11T07:29:55,338 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82d8c2e6825b414d93b6edd0c287638d, keycount=300, bloomtype=ROW, size=55.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731310190563 2024-11-11T07:29:55,338 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ebd3ac1149a4e13887f440e2ae79e92, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731310190730 2024-11-11T07:29:55,338 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d3cfd10d43d4010905550500b6f04db, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731310191961 2024-11-11T07:29:55,339 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9d9421655cc446f920930dc2adb32cf, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731310191961 2024-11-11T07:29:55,349 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#B#compaction#123 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:55,349 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/1c3dcc1b822349e0a3760c938e814e1b is 50, key is test_row_0/B:col10/1731310191973/Put/seqid=0 2024-11-11T07:29:55,352 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:29:55,363 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111905ace1a1457457d89998cf029fd6b82_18312f0c839a758e8897bb39c81d93ff store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:29:55,369 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111905ace1a1457457d89998cf029fd6b82_18312f0c839a758e8897bb39c81d93ff, store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:29:55,369 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111905ace1a1457457d89998cf029fd6b82_18312f0c839a758e8897bb39c81d93ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:29:55,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-11T07:29:55,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:55,373 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:29:55,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:29:55,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:55,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:29:55,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:55,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:29:55,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:55,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T07:29:55,400 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741966_1142 (size=12104) 2024-11-11T07:29:55,407 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/1c3dcc1b822349e0a3760c938e814e1b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/1c3dcc1b822349e0a3760c938e814e1b 2024-11-11T07:29:55,413 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/B of 18312f0c839a758e8897bb39c81d93ff into 1c3dcc1b822349e0a3760c938e814e1b(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:55,413 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:29:55,413 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/B, priority=13, startTime=1731310195336; duration=0sec 2024-11-11T07:29:55,414 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:55,414 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:B 2024-11-11T07:29:55,414 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:55,415 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:55,415 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/C is initiating minor compaction (all files) 2024-11-11T07:29:55,415 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/C in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:55,415 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/9104fce7b44b463ab9f1c730173748ee, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/692a372fa9d1456da7db589eb99874c4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/1cdb0eead393426ca43ca26c402923e4] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=35.2 K 2024-11-11T07:29:55,416 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 9104fce7b44b463ab9f1c730173748ee, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731310190584 2024-11-11T07:29:55,416 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 692a372fa9d1456da7db589eb99874c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731310190730 2024-11-11T07:29:55,416 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cdb0eead393426ca43ca26c402923e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731310191961 2024-11-11T07:29:55,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:55,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:29:55,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ece2b2dc25654321bb743af5e2c9eb80_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310194246/Put/seqid=0 2024-11-11T07:29:55,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310255446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:55,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:55,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310255446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:55,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,459 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:55,461 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:55,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310255450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:55,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,462 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:55,462 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:55,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310255452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:55,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:55,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:55,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310255454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:55,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,465 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:55,467 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#C#compaction#126 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:55,468 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/79898e5d0dfe44ea850abda2610d8379 is 50, key is test_row_0/C:col10/1731310191973/Put/seqid=0 2024-11-11T07:29:55,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741967_1143 (size=4469) 2024-11-11T07:29:55,470 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#A#compaction#124 average throughput is 0.21 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:55,471 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/b52402183a184fce929716b91a3ed5e9 is 175, key is test_row_0/A:col10/1731310191973/Put/seqid=0 2024-11-11T07:29:55,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741968_1144 (size=12154) 2024-11-11T07:29:55,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741969_1145 (size=12104) 2024-11-11T07:29:55,532 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/79898e5d0dfe44ea850abda2610d8379 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/79898e5d0dfe44ea850abda2610d8379 2024-11-11T07:29:55,545 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/C of 18312f0c839a758e8897bb39c81d93ff into 79898e5d0dfe44ea850abda2610d8379(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:55,545 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:29:55,545 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/C, priority=13, startTime=1731310195336; duration=0sec 2024-11-11T07:29:55,545 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:55,545 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:C 2024-11-11T07:29:55,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741970_1146 (size=31058) 2024-11-11T07:29:55,558 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/b52402183a184fce929716b91a3ed5e9 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/b52402183a184fce929716b91a3ed5e9 2024-11-11T07:29:55,569 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/A of 18312f0c839a758e8897bb39c81d93ff into b52402183a184fce929716b91a3ed5e9(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:55,569 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:29:55,569 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/A, priority=13, startTime=1731310195335; duration=0sec 2024-11-11T07:29:55,569 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:55,569 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:A 2024-11-11T07:29:55,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:55,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310255565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:55,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310255565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:55,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:55,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310255565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,576 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:55,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:55,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310255565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:55,582 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310255576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:55,583 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:55,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310255784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:55,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:55,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310255784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:55,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,800 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:55,806 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:55,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310255795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:55,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:55,787 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,806 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310255785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:55,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310255795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:55,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T07:29:55,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:55,934 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ece2b2dc25654321bb743af5e2c9eb80_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ece2b2dc25654321bb743af5e2c9eb80_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:55,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/85db1f5653cc46a78e32176521a5ae02, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:29:55,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/85db1f5653cc46a78e32176521a5ae02 is 175, key is test_row_0/A:col10/1731310194246/Put/seqid=0 2024-11-11T07:29:55,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741971_1147 (size=30955) 2024-11-11T07:29:56,105 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:56,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310256105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,106 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,107 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:56,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310256115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:56,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:56,119 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310256115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:56,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310256115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:56,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:56,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310256115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:56,120 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,121 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:56,348 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/85db1f5653cc46a78e32176521a5ae02 2024-11-11T07:29:56,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/347fc3c22816425c8be6363fd174f1a7 is 50, key is test_row_0/B:col10/1731310194246/Put/seqid=0 2024-11-11T07:29:56,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741972_1148 (size=12001) 2024-11-11T07:29:56,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310256616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:56,624 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310256624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:56,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310256634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:56,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:56,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310256635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:56,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310256635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,645 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:56,650 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:56,843 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/347fc3c22816425c8be6363fd174f1a7 2024-11-11T07:29:56,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/0676dd95de714ee1820f90c6bf3b82c8 is 50, key is test_row_0/C:col10/1731310194246/Put/seqid=0 2024-11-11T07:29:56,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T07:29:56,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741973_1149 (size=12001) 2024-11-11T07:29:56,905 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/0676dd95de714ee1820f90c6bf3b82c8 2024-11-11T07:29:56,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/85db1f5653cc46a78e32176521a5ae02 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/85db1f5653cc46a78e32176521a5ae02 2024-11-11T07:29:56,925 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/85db1f5653cc46a78e32176521a5ae02, entries=150, sequenceid=83, filesize=30.2 K 2024-11-11T07:29:56,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/347fc3c22816425c8be6363fd174f1a7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/347fc3c22816425c8be6363fd174f1a7 2024-11-11T07:29:56,939 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/347fc3c22816425c8be6363fd174f1a7, entries=150, sequenceid=83, filesize=11.7 K 2024-11-11T07:29:56,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/0676dd95de714ee1820f90c6bf3b82c8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/0676dd95de714ee1820f90c6bf3b82c8 2024-11-11T07:29:56,946 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/0676dd95de714ee1820f90c6bf3b82c8, entries=150, sequenceid=83, filesize=11.7 K 2024-11-11T07:29:56,950 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 18312f0c839a758e8897bb39c81d93ff in 1577ms, sequenceid=83, compaction requested=false 2024-11-11T07:29:56,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:29:56,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:56,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-11T07:29:56,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-11T07:29:56,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-11T07:29:56,953 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2060 sec 2024-11-11T07:29:56,959 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.2170 sec 2024-11-11T07:29:57,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:57,642 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T07:29:57,646 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:29:57,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:57,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:29:57,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:57,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:29:57,647 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:57,661 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116b7412820f07445599c3ab0867132684_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310195439/Put/seqid=0 2024-11-11T07:29:57,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:57,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310257681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:57,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,692 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:57,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:57,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310257681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:57,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:57,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310257687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,696 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:57,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310257694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:57,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741974_1150 (size=14594) 2024-11-11T07:29:57,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:57,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310257699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:57,714 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:57,719 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116b7412820f07445599c3ab0867132684_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116b7412820f07445599c3ab0867132684_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:57,723 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/03d20c5fc85547698aaaf983cbee57ed, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:29:57,723 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/03d20c5fc85547698aaaf983cbee57ed is 175, key is test_row_0/A:col10/1731310195439/Put/seqid=0 2024-11-11T07:29:57,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741975_1151 (size=39549) 2024-11-11T07:29:57,750 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=100, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/03d20c5fc85547698aaaf983cbee57ed 2024-11-11T07:29:57,773 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/2ea2e13b6f3d47aaa75b2bc7faba38e4 is 50, key is test_row_0/B:col10/1731310195439/Put/seqid=0 2024-11-11T07:29:57,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741976_1152 (size=12001) 2024-11-11T07:29:57,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310257794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:57,798 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:57,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310257805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:57,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310257805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,814 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:57,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:57,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310257805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,815 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:57,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:57,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310257815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:57,825 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310258006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,030 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310258024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310258025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,031 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310258025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310258035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/2ea2e13b6f3d47aaa75b2bc7faba38e4 2024-11-11T07:29:58,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/218a10e1b58147428ab247fa9f0c4642 is 50, key is test_row_0/C:col10/1731310195439/Put/seqid=0 2024-11-11T07:29:58,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741977_1153 (size=12001) 2024-11-11T07:29:58,267 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/218a10e1b58147428ab247fa9f0c4642 2024-11-11T07:29:58,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/03d20c5fc85547698aaaf983cbee57ed as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/03d20c5fc85547698aaaf983cbee57ed 2024-11-11T07:29:58,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/03d20c5fc85547698aaaf983cbee57ed, entries=200, sequenceid=100, filesize=38.6 K 2024-11-11T07:29:58,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/2ea2e13b6f3d47aaa75b2bc7faba38e4 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/2ea2e13b6f3d47aaa75b2bc7faba38e4 2024-11-11T07:29:58,296 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/2ea2e13b6f3d47aaa75b2bc7faba38e4, entries=150, sequenceid=100, filesize=11.7 K 2024-11-11T07:29:58,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/218a10e1b58147428ab247fa9f0c4642 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/218a10e1b58147428ab247fa9f0c4642 2024-11-11T07:29:58,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/218a10e1b58147428ab247fa9f0c4642, entries=150, sequenceid=100, filesize=11.7 K 2024-11-11T07:29:58,319 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 18312f0c839a758e8897bb39c81d93ff in 676ms, sequenceid=100, compaction requested=true 2024-11-11T07:29:58,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:29:58,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:29:58,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:58,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:29:58,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:29:58,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:29:58,319 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T07:29:58,319 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:58,320 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:58,320 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/A is initiating minor compaction (all files) 2024-11-11T07:29:58,320 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/A in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:58,321 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/b52402183a184fce929716b91a3ed5e9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/85db1f5653cc46a78e32176521a5ae02, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/03d20c5fc85547698aaaf983cbee57ed] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=99.2 K 2024-11-11T07:29:58,321 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:58,321 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/b52402183a184fce929716b91a3ed5e9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/85db1f5653cc46a78e32176521a5ae02, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/03d20c5fc85547698aaaf983cbee57ed] 2024-11-11T07:29:58,321 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting b52402183a184fce929716b91a3ed5e9, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731310191961 2024-11-11T07:29:58,322 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85db1f5653cc46a78e32176521a5ae02, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731310194246 2024-11-11T07:29:58,322 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03d20c5fc85547698aaaf983cbee57ed, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731310195439 2024-11-11T07:29:58,322 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:58,331 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:29:58,333 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T07:29:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:29:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:29:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:29:58,333 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:58,334 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:58,335 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/C is initiating minor compaction (all files) 2024-11-11T07:29:58,335 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/C in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:58,335 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/79898e5d0dfe44ea850abda2610d8379, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/0676dd95de714ee1820f90c6bf3b82c8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/218a10e1b58147428ab247fa9f0c4642] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=35.3 K 2024-11-11T07:29:58,335 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 79898e5d0dfe44ea850abda2610d8379, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731310191961 2024-11-11T07:29:58,336 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 0676dd95de714ee1820f90c6bf3b82c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731310194246 2024-11-11T07:29:58,336 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 218a10e1b58147428ab247fa9f0c4642, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731310195439 2024-11-11T07:29:58,339 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411111dbe6b9fbe364329ad8e72f1710c8cc3_18312f0c839a758e8897bb39c81d93ff store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:29:58,345 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411111dbe6b9fbe364329ad8e72f1710c8cc3_18312f0c839a758e8897bb39c81d93ff, store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:29:58,346 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111dbe6b9fbe364329ad8e72f1710c8cc3_18312f0c839a758e8897bb39c81d93ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:29:58,353 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#C#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:58,354 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/d564f4a5b62b4280b3e893547cf4fa3c is 50, key is test_row_0/C:col10/1731310195439/Put/seqid=0 2024-11-11T07:29:58,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113786f4291ed24041a135a8a49905c977_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310198326/Put/seqid=0 2024-11-11T07:29:58,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741978_1154 (size=4469) 2024-11-11T07:29:58,377 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#A#compaction#132 average throughput is 0.53 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:58,378 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/6e417d57a4274f4daf0944e6b0ac35dd is 175, key is test_row_0/A:col10/1731310195439/Put/seqid=0 2024-11-11T07:29:58,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310258365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,380 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310258367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,383 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310258379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,388 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310258380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310258381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,389 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741980_1156 (size=12154) 2024-11-11T07:29:58,393 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:58,397 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411113786f4291ed24041a135a8a49905c977_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113786f4291ed24041a135a8a49905c977_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:58,400 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/e5d95791a6334f38a96b2a163c6686a1, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:29:58,400 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/e5d95791a6334f38a96b2a163c6686a1 is 175, key is test_row_0/A:col10/1731310198326/Put/seqid=0 2024-11-11T07:29:58,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741979_1155 (size=12207) 2024-11-11T07:29:58,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741981_1157 (size=31161) 2024-11-11T07:29:58,432 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/d564f4a5b62b4280b3e893547cf4fa3c as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/d564f4a5b62b4280b3e893547cf4fa3c 2024-11-11T07:29:58,450 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/6e417d57a4274f4daf0944e6b0ac35dd as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/6e417d57a4274f4daf0944e6b0ac35dd 2024-11-11T07:29:58,453 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/C of 18312f0c839a758e8897bb39c81d93ff into d564f4a5b62b4280b3e893547cf4fa3c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:58,454 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:29:58,454 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/C, priority=13, startTime=1731310198319; duration=0sec 2024-11-11T07:29:58,456 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:29:58,456 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:C 2024-11-11T07:29:58,456 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:29:58,458 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:29:58,458 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/B is initiating minor compaction (all files) 2024-11-11T07:29:58,458 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/B in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:58,458 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/A of 18312f0c839a758e8897bb39c81d93ff into 6e417d57a4274f4daf0944e6b0ac35dd(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:58,458 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:29:58,458 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/A, priority=13, startTime=1731310198319; duration=0sec 2024-11-11T07:29:58,458 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:58,459 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:A 2024-11-11T07:29:58,462 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/1c3dcc1b822349e0a3760c938e814e1b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/347fc3c22816425c8be6363fd174f1a7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/2ea2e13b6f3d47aaa75b2bc7faba38e4] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=35.3 K 2024-11-11T07:29:58,463 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c3dcc1b822349e0a3760c938e814e1b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731310191961 2024-11-11T07:29:58,464 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 347fc3c22816425c8be6363fd174f1a7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731310194246 2024-11-11T07:29:58,464 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ea2e13b6f3d47aaa75b2bc7faba38e4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731310195439 2024-11-11T07:29:58,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741982_1158 (size=30955) 2024-11-11T07:29:58,497 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/e5d95791a6334f38a96b2a163c6686a1 2024-11-11T07:29:58,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310258485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,498 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310258495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310258495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310258495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310258495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,502 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,506 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#B#compaction#135 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:29:58,507 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/873b065458e346439380207534fc51cf is 50, key is test_row_0/B:col10/1731310195439/Put/seqid=0 2024-11-11T07:29:58,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/732352b7793848c48ee5c82821ede75b is 50, key is test_row_0/B:col10/1731310198326/Put/seqid=0 2024-11-11T07:29:58,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741983_1159 (size=12207) 2024-11-11T07:29:58,551 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/873b065458e346439380207534fc51cf as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/873b065458e346439380207534fc51cf 2024-11-11T07:29:58,557 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/B of 18312f0c839a758e8897bb39c81d93ff into 873b065458e346439380207534fc51cf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:29:58,557 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:29:58,557 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/B, priority=13, startTime=1731310198319; duration=0sec 2024-11-11T07:29:58,557 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:29:58,558 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:B 2024-11-11T07:29:58,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741984_1160 (size=12001) 2024-11-11T07:29:58,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/732352b7793848c48ee5c82821ede75b 2024-11-11T07:29:58,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/f50c3e48aeb14365a814171f879751e9 is 50, key is test_row_0/C:col10/1731310198326/Put/seqid=0 2024-11-11T07:29:58,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741985_1161 (size=12001) 2024-11-11T07:29:58,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310258705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310258705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310258705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,709 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310258705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,710 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310258714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:58,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:58,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-11T07:29:58,907 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:29:58,911 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:29:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-11T07:29:58,914 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:29:58,915 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:29:58,915 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:29:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T07:29:58,927 INFO [master/ce2cf0244921:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T07:29:58,927 INFO [master/ce2cf0244921:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T07:29:59,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310259015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310259015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,019 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310259016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310259016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T07:29:59,036 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310259034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/f50c3e48aeb14365a814171f879751e9 2024-11-11T07:29:59,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/e5d95791a6334f38a96b2a163c6686a1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/e5d95791a6334f38a96b2a163c6686a1 2024-11-11T07:29:59,056 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/e5d95791a6334f38a96b2a163c6686a1, entries=150, sequenceid=119, filesize=30.2 K 2024-11-11T07:29:59,058 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/732352b7793848c48ee5c82821ede75b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/732352b7793848c48ee5c82821ede75b 2024-11-11T07:29:59,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T07:29:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:29:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:59,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:59,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:59,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:29:59,071 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/732352b7793848c48ee5c82821ede75b, entries=150, sequenceid=119, filesize=11.7 K 2024-11-11T07:29:59,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/f50c3e48aeb14365a814171f879751e9 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/f50c3e48aeb14365a814171f879751e9 2024-11-11T07:29:59,092 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/f50c3e48aeb14365a814171f879751e9, entries=150, sequenceid=119, filesize=11.7 K 2024-11-11T07:29:59,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 18312f0c839a758e8897bb39c81d93ff in 762ms, sequenceid=119, compaction requested=false 2024-11-11T07:29:59,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:29:59,226 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-11T07:29:59,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:29:59,227 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T07:29:59,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:29:59,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:59,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:29:59,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:59,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:29:59,227 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:29:59,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T07:29:59,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111cc3be004d8b4084a261bf58920a9686_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310198376/Put/seqid=0 2024-11-11T07:29:59,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741986_1162 (size=12254) 2024-11-11T07:29:59,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:29:59,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:59,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T07:29:59,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310259571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310259571, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,582 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310259575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,583 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,584 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310259579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,585 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310259583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,591 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:29:59,690 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111cc3be004d8b4084a261bf58920a9686_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111cc3be004d8b4084a261bf58920a9686_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:29:59,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/ac838d5e38f146fa9cc211ed9b73477e, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:29:59,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/ac838d5e38f146fa9cc211ed9b73477e is 175, key is test_row_0/A:col10/1731310198376/Put/seqid=0 2024-11-11T07:29:59,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310259684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,695 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310259684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310259696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310259696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,708 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310259702, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,710 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741987_1163 (size=31055) 2024-11-11T07:29:59,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310259905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,916 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310259905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310259915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310259915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,925 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,925 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:29:59,925 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:29:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310259915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:29:59,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:29:59,926 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T07:30:00,142 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=136, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/ac838d5e38f146fa9cc211ed9b73477e 2024-11-11T07:30:00,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/cb947a4ef75741f5bd2f59576b2c4294 is 50, key is test_row_0/B:col10/1731310198376/Put/seqid=0 2024-11-11T07:30:00,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741988_1164 (size=12101) 2024-11-11T07:30:00,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:00,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310260226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:00,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:00,232 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:00,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310260226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:00,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:00,240 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:00,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310260238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:00,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:00,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:00,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310260238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:00,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,242 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:00,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310260239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:00,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,243 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:00,605 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/cb947a4ef75741f5bd2f59576b2c4294 2024-11-11T07:30:00,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/88b3852a82eb4ef3ad9696c3160b60d1 is 50, key is test_row_0/C:col10/1731310198376/Put/seqid=0 2024-11-11T07:30:00,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741989_1165 (size=12101) 2024-11-11T07:30:00,694 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/88b3852a82eb4ef3ad9696c3160b60d1 2024-11-11T07:30:00,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/ac838d5e38f146fa9cc211ed9b73477e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/ac838d5e38f146fa9cc211ed9b73477e 2024-11-11T07:30:00,706 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/ac838d5e38f146fa9cc211ed9b73477e, entries=150, sequenceid=136, filesize=30.3 K 2024-11-11T07:30:00,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/cb947a4ef75741f5bd2f59576b2c4294 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/cb947a4ef75741f5bd2f59576b2c4294 2024-11-11T07:30:00,720 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/cb947a4ef75741f5bd2f59576b2c4294, entries=150, sequenceid=136, filesize=11.8 K 2024-11-11T07:30:00,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/88b3852a82eb4ef3ad9696c3160b60d1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/88b3852a82eb4ef3ad9696c3160b60d1 2024-11-11T07:30:00,728 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/88b3852a82eb4ef3ad9696c3160b60d1, entries=150, sequenceid=136, filesize=11.8 K 2024-11-11T07:30:00,729 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 18312f0c839a758e8897bb39c81d93ff in 1502ms, sequenceid=136, compaction requested=true 2024-11-11T07:30:00,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:00,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:00,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-11T07:30:00,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-11T07:30:00,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-11T07:30:00,738 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8200 sec 2024-11-11T07:30:00,752 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T07:30:00,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:30:00,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:00,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:30:00,752 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:00,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:30:00,753 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:00,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:00,756 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.8360 sec 2024-11-11T07:30:00,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111145e612c2cbbc429b999145e93c546001_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310199569/Put/seqid=0 2024-11-11T07:30:00,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741990_1166 (size=12304) 2024-11-11T07:30:00,803 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:00,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:00,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310260785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:00,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310260787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:00,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:00,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310260791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:00,812 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111145e612c2cbbc429b999145e93c546001_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111145e612c2cbbc429b999145e93c546001_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:00,814 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/8af59a589ae44a3b89db942bd475d100, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:00,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/8af59a589ae44a3b89db942bd475d100 is 175, key is test_row_0/A:col10/1731310199569/Put/seqid=0 2024-11-11T07:30:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:00,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:00,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310260804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:00,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:00,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310260804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,817 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:00,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741991_1167 (size=31105) 2024-11-11T07:30:00,858 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/8af59a589ae44a3b89db942bd475d100 2024-11-11T07:30:00,868 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/28450c8d06b44c63b71c1ca18289f84a is 50, key is test_row_0/B:col10/1731310199569/Put/seqid=0 2024-11-11T07:30:00,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:00,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310260914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,926 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:00,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:00,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310260916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,927 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:00,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741992_1168 (size=12151) 2024-11-11T07:30:00,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/28450c8d06b44c63b71c1ca18289f84a 2024-11-11T07:30:00,938 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:00,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310260925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,939 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:00,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:00,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310260925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:00,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:00,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310260925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:00,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:00,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:00,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/e5124c2a0ba9441caacabda265eb598d is 50, key is test_row_0/C:col10/1731310199569/Put/seqid=0 2024-11-11T07:30:00,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741993_1169 (size=12151) 2024-11-11T07:30:00,997 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/e5124c2a0ba9441caacabda265eb598d 2024-11-11T07:30:01,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/8af59a589ae44a3b89db942bd475d100 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/8af59a589ae44a3b89db942bd475d100 2024-11-11T07:30:01,008 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/8af59a589ae44a3b89db942bd475d100, entries=150, sequenceid=155, filesize=30.4 K 2024-11-11T07:30:01,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/28450c8d06b44c63b71c1ca18289f84a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/28450c8d06b44c63b71c1ca18289f84a 2024-11-11T07:30:01,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/28450c8d06b44c63b71c1ca18289f84a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T07:30:01,018 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/e5124c2a0ba9441caacabda265eb598d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/e5124c2a0ba9441caacabda265eb598d 2024-11-11T07:30:01,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/e5124c2a0ba9441caacabda265eb598d, entries=150, sequenceid=155, filesize=11.9 K 2024-11-11T07:30:01,026 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 18312f0c839a758e8897bb39c81d93ff in 274ms, sequenceid=155, compaction requested=true 2024-11-11T07:30:01,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:01,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:01,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:01,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:01,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:30:01,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:01,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T07:30:01,027 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:01,029 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 124276 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:01,029 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/A is initiating minor compaction (all files) 2024-11-11T07:30:01,029 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/A in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:01,029 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/6e417d57a4274f4daf0944e6b0ac35dd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/e5d95791a6334f38a96b2a163c6686a1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/ac838d5e38f146fa9cc211ed9b73477e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/8af59a589ae44a3b89db942bd475d100] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=121.4 K 2024-11-11T07:30:01,029 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:01,029 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/6e417d57a4274f4daf0944e6b0ac35dd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/e5d95791a6334f38a96b2a163c6686a1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/ac838d5e38f146fa9cc211ed9b73477e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/8af59a589ae44a3b89db942bd475d100] 2024-11-11T07:30:01,030 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e417d57a4274f4daf0944e6b0ac35dd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731310195439 2024-11-11T07:30:01,030 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:01,030 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting e5d95791a6334f38a96b2a163c6686a1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731310197685 2024-11-11T07:30:01,033 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting ac838d5e38f146fa9cc211ed9b73477e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731310198358 2024-11-11T07:30:01,034 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 8af59a589ae44a3b89db942bd475d100, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731310199569 2024-11-11T07:30:01,054 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:01,055 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/C is initiating minor compaction (all files) 2024-11-11T07:30:01,055 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/C in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:01,055 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/d564f4a5b62b4280b3e893547cf4fa3c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/f50c3e48aeb14365a814171f879751e9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/88b3852a82eb4ef3ad9696c3160b60d1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/e5124c2a0ba9441caacabda265eb598d] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=47.3 K 2024-11-11T07:30:01,055 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:01,056 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting d564f4a5b62b4280b3e893547cf4fa3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731310195439 2024-11-11T07:30:01,058 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting f50c3e48aeb14365a814171f879751e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731310197685 2024-11-11T07:30:01,060 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88b3852a82eb4ef3ad9696c3160b60d1, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731310198358 2024-11-11T07:30:01,060 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting e5124c2a0ba9441caacabda265eb598d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731310199569 2024-11-11T07:30:01,070 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111b19dafbfdb684dc98bf2d57111f40c78_18312f0c839a758e8897bb39c81d93ff store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:01,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-11T07:30:01,075 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:01,075 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111b19dafbfdb684dc98bf2d57111f40c78_18312f0c839a758e8897bb39c81d93ff, store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:01,075 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b19dafbfdb684dc98bf2d57111f40c78_18312f0c839a758e8897bb39c81d93ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:01,082 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:01,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-11T07:30:01,090 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#C#compaction#145 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:01,090 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/5691605df52c469eb114daa05a752408 is 50, key is test_row_0/C:col10/1731310199569/Put/seqid=0 2024-11-11T07:30:01,092 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:01,093 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:01,093 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:01,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T07:30:01,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:01,143 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T07:30:01,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:30:01,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:01,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:30:01,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:01,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:30:01,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:01,161 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741994_1170 (size=4469) 2024-11-11T07:30:01,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741995_1171 (size=12493) 2024-11-11T07:30:01,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a29bd16327d8417a877fe42d89e13dc3_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310201141/Put/seqid=0 2024-11-11T07:30:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T07:30:01,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310261187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,205 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,207 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310261195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310261197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310261204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,214 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310261207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,214 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,217 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741996_1172 (size=14794) 2024-11-11T07:30:01,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,218 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,222 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:01,228 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a29bd16327d8417a877fe42d89e13dc3_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a29bd16327d8417a877fe42d89e13dc3_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:01,230 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/8d13eb3f76c0471d864f09d9ce4762c7, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:01,230 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/8d13eb3f76c0471d864f09d9ce4762c7 is 175, key is test_row_0/A:col10/1731310201141/Put/seqid=0 2024-11-11T07:30:01,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:01,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:01,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:01,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:01,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:01,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:01,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:01,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741997_1173 (size=39749) 2024-11-11T07:30:01,261 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/8d13eb3f76c0471d864f09d9ce4762c7 2024-11-11T07:30:01,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/27869486f23d431da4dd204119d4f69a is 50, key is test_row_0/B:col10/1731310201141/Put/seqid=0 2024-11-11T07:30:01,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310261315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,322 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741998_1174 (size=12151) 2024-11-11T07:30:01,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310261324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310261325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310261325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,339 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310261326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T07:30:01,419 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:01,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:01,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:01,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:01,420 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:01,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:01,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:01,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310261524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,555 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310261545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310261545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,556 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,556 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,557 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310261547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310261555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,569 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,571 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#A#compaction#144 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:01,574 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/12ab330da38d4780bb6d4ab1e6ae2178 is 175, key is test_row_0/A:col10/1731310199569/Put/seqid=0 2024-11-11T07:30:01,599 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/5691605df52c469eb114daa05a752408 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/5691605df52c469eb114daa05a752408 2024-11-11T07:30:01,599 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:01,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:01,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:01,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:01,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:01,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:01,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:01,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741999_1175 (size=31447) 2024-11-11T07:30:01,643 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/C of 18312f0c839a758e8897bb39c81d93ff into 5691605df52c469eb114daa05a752408(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:01,644 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:01,644 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/C, priority=12, startTime=1731310201027; duration=0sec 2024-11-11T07:30:01,644 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:01,644 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:C 2024-11-11T07:30:01,644 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:01,649 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:01,649 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/B is initiating minor compaction (all files) 2024-11-11T07:30:01,649 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/B in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:01,650 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/873b065458e346439380207534fc51cf, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/732352b7793848c48ee5c82821ede75b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/cb947a4ef75741f5bd2f59576b2c4294, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/28450c8d06b44c63b71c1ca18289f84a] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=47.3 K 2024-11-11T07:30:01,651 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 873b065458e346439380207534fc51cf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731310195439 2024-11-11T07:30:01,652 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 732352b7793848c48ee5c82821ede75b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731310197685 2024-11-11T07:30:01,653 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb947a4ef75741f5bd2f59576b2c4294, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731310198358 2024-11-11T07:30:01,657 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28450c8d06b44c63b71c1ca18289f84a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731310199569 2024-11-11T07:30:01,657 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/12ab330da38d4780bb6d4ab1e6ae2178 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/12ab330da38d4780bb6d4ab1e6ae2178 2024-11-11T07:30:01,680 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/A of 18312f0c839a758e8897bb39c81d93ff into 12ab330da38d4780bb6d4ab1e6ae2178(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:01,680 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:01,680 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/A, priority=12, startTime=1731310201026; duration=0sec 2024-11-11T07:30:01,680 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#B#compaction#148 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:01,680 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/603fe0ed84f14777a82e5c11e362b957 is 50, key is test_row_0/B:col10/1731310199569/Put/seqid=0 2024-11-11T07:30:01,681 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:01,681 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:A 2024-11-11T07:30:01,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742000_1176 (size=12493) 2024-11-11T07:30:01,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T07:30:01,729 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/27869486f23d431da4dd204119d4f69a 2024-11-11T07:30:01,756 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/ef21f379dd334c23beb2300102a3342c is 50, key is test_row_0/C:col10/1731310201141/Put/seqid=0 2024-11-11T07:30:01,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:01,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:01,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:01,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:01,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:01,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:01,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:01,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742001_1177 (size=12151) 2024-11-11T07:30:01,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310261851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,866 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,867 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310261865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,882 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310261865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310261866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:01,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310261875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:01,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,884 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:01,886 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:01,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:01,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:01,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:01,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:01,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:01,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:01,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:02,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:02,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:02,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:02,117 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,117 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,131 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/603fe0ed84f14777a82e5c11e362b957 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/603fe0ed84f14777a82e5c11e362b957 2024-11-11T07:30:02,146 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/B of 18312f0c839a758e8897bb39c81d93ff into 603fe0ed84f14777a82e5c11e362b957(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:02,146 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:02,146 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/B, priority=12, startTime=1731310201026; duration=0sec 2024-11-11T07:30:02,146 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:02,146 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:B 2024-11-11T07:30:02,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/ef21f379dd334c23beb2300102a3342c 2024-11-11T07:30:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T07:30:02,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:02,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:02,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:02,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:02,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/8d13eb3f76c0471d864f09d9ce4762c7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/8d13eb3f76c0471d864f09d9ce4762c7 2024-11-11T07:30:02,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/8d13eb3f76c0471d864f09d9ce4762c7, entries=200, sequenceid=169, filesize=38.8 K 2024-11-11T07:30:02,299 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/27869486f23d431da4dd204119d4f69a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/27869486f23d431da4dd204119d4f69a 2024-11-11T07:30:02,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/27869486f23d431da4dd204119d4f69a, entries=150, sequenceid=169, filesize=11.9 K 2024-11-11T07:30:02,319 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/ef21f379dd334c23beb2300102a3342c as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/ef21f379dd334c23beb2300102a3342c 2024-11-11T07:30:02,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/ef21f379dd334c23beb2300102a3342c, entries=150, sequenceid=169, filesize=11.9 K 2024-11-11T07:30:02,358 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 18312f0c839a758e8897bb39c81d93ff in 1216ms, sequenceid=169, compaction requested=false 2024-11-11T07:30:02,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:02,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T07:30:02,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:30:02,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:02,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:30:02,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:02,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:30:02,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:02,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111170953c139cd14a7a8e0fd61b8bc8c47d_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310201197/Put/seqid=0 2024-11-11T07:30:02,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:02,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310262449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,456 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:02,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310262450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,456 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,457 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:02,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:02,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310262456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:02,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,464 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:02,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742002_1178 (size=14794) 2024-11-11T07:30:02,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:02,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:02,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:02,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:02,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,473 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310262470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:02,474 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:02,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310262470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:02,474 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:02,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:02,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310262564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:02,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:02,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:02,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310262565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:02,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,569 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:02,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:02,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310262575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:02,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:02,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:02,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310262587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,594 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:02,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:02,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310262586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,597 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:02,634 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:02,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:02,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:02,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:02,635 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,776 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310262776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:02,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,777 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:02,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:02,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310262776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:02,777 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,778 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:02,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:02,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:02,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:02,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:02,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:02,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310262784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:02,813 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:02,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310262804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:02,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:02,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:02,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310262807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:02,816 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:02,871 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:02,889 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111170953c139cd14a7a8e0fd61b8bc8c47d_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111170953c139cd14a7a8e0fd61b8bc8c47d_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:02,899 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/140ce797b527429ba679afdf806f89aa, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:02,899 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/140ce797b527429ba679afdf806f89aa is 175, key is test_row_0/A:col10/1731310201197/Put/seqid=0 2024-11-11T07:30:02,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742003_1179 (size=39749) 2024-11-11T07:30:02,927 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=191, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/140ce797b527429ba679afdf806f89aa 2024-11-11T07:30:02,940 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:02,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:02,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:02,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:02,941 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:02,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/98dff9632b7043b7a4dd28bc9bb41d6f is 50, key is test_row_0/B:col10/1731310201197/Put/seqid=0 2024-11-11T07:30:03,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742004_1180 (size=12151) 2024-11-11T07:30:03,011 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/98dff9632b7043b7a4dd28bc9bb41d6f 2024-11-11T07:30:03,034 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/879e20cbe2e84fb4824810bed58e1ed5 is 50, key is test_row_0/C:col10/1731310201197/Put/seqid=0 2024-11-11T07:30:03,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742005_1181 (size=12151) 2024-11-11T07:30:03,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310263084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:03,091 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310263084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:03,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/879e20cbe2e84fb4824810bed58e1ed5 2024-11-11T07:30:03,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,093 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:03,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,093 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:03,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:03,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:03,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:03,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:03,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/140ce797b527429ba679afdf806f89aa as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/140ce797b527429ba679afdf806f89aa 2024-11-11T07:30:03,114 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/140ce797b527429ba679afdf806f89aa, entries=200, sequenceid=191, filesize=38.8 K 2024-11-11T07:30:03,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:03,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310263107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:03,115 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/98dff9632b7043b7a4dd28bc9bb41d6f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/98dff9632b7043b7a4dd28bc9bb41d6f 2024-11-11T07:30:03,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/98dff9632b7043b7a4dd28bc9bb41d6f, entries=150, sequenceid=191, filesize=11.9 K 2024-11-11T07:30:03,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/879e20cbe2e84fb4824810bed58e1ed5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/879e20cbe2e84fb4824810bed58e1ed5 2024-11-11T07:30:03,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:03,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310263127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:03,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/879e20cbe2e84fb4824810bed58e1ed5, entries=150, sequenceid=191, filesize=11.9 K 2024-11-11T07:30:03,137 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 18312f0c839a758e8897bb39c81d93ff in 737ms, sequenceid=191, compaction requested=true 2024-11-11T07:30:03,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:03,137 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:03,138 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110945 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:03,138 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/A is initiating minor compaction (all files) 2024-11-11T07:30:03,138 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/A in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:03,138 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/12ab330da38d4780bb6d4ab1e6ae2178, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/8d13eb3f76c0471d864f09d9ce4762c7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/140ce797b527429ba679afdf806f89aa] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=108.3 K 2024-11-11T07:30:03,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,138 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:03,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,139 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:03,139 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/12ab330da38d4780bb6d4ab1e6ae2178, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/8d13eb3f76c0471d864f09d9ce4762c7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/140ce797b527429ba679afdf806f89aa] 2024-11-11T07:30:03,139 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12ab330da38d4780bb6d4ab1e6ae2178, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731310199569 2024-11-11T07:30:03,140 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d13eb3f76c0471d864f09d9ce4762c7, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731310200780 2024-11-11T07:30:03,140 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 140ce797b527429ba679afdf806f89aa, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731310201191 2024-11-11T07:30:03,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:03,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:03,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:03,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:03,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:03,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:30:03,144 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:03,151 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:03,154 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:03,155 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/B is initiating minor compaction (all files) 2024-11-11T07:30:03,155 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/B in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:03,155 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/603fe0ed84f14777a82e5c11e362b957, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/27869486f23d431da4dd204119d4f69a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/98dff9632b7043b7a4dd28bc9bb41d6f] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=35.9 K 2024-11-11T07:30:03,158 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 603fe0ed84f14777a82e5c11e362b957, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731310199569 2024-11-11T07:30:03,159 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 27869486f23d431da4dd204119d4f69a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731310200780 2024-11-11T07:30:03,159 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411110a6e0d3be0654485b0f2007662963554_18312f0c839a758e8897bb39c81d93ff store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:03,161 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411110a6e0d3be0654485b0f2007662963554_18312f0c839a758e8897bb39c81d93ff, store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:03,162 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110a6e0d3be0654485b0f2007662963554_18312f0c839a758e8897bb39c81d93ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:03,162 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 98dff9632b7043b7a4dd28bc9bb41d6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731310201191 2024-11-11T07:30:03,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T07:30:03,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:30:03,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:03,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:30:03,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:03,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:30:03,164 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:03,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:03,209 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#B#compaction#154 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:03,209 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/a732fe8d1b4040e293705d2bc233734a is 50, key is test_row_0/B:col10/1731310201197/Put/seqid=0 2024-11-11T07:30:03,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742006_1182 (size=4469) 2024-11-11T07:30:03,218 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#A#compaction#153 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:03,219 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/5188f9e66a2d4bd1a6650e387863a122 is 175, key is test_row_0/A:col10/1731310201197/Put/seqid=0 2024-11-11T07:30:03,225 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111182c307212c2c4d82a71bf2ecb2f0e81a_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310203136/Put/seqid=0 2024-11-11T07:30:03,249 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:03,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:03,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:03,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:03,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T07:30:03,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742009_1185 (size=14794) 2024-11-11T07:30:03,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742008_1184 (size=31549) 2024-11-11T07:30:03,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742007_1183 (size=12595) 2024-11-11T07:30:03,371 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:03,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310263368, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:03,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,372 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:03,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:03,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:03,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:03,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:03,411 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:03,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310263474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:03,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,475 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:03,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:03,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:03,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:03,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:03,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310263605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:03,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:03,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310263605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,631 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:03,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:03,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310263633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,636 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:03,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:03,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310263652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:03,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,666 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:03,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:03,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310263687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:03,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:03,739 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:03,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:03,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:03,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:03,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:03,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,786 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111182c307212c2c4d82a71bf2ecb2f0e81a_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111182c307212c2c4d82a71bf2ecb2f0e81a_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:03,791 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/4025bc6b0eb64f26bdd00b4beb44c896, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:03,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/4025bc6b0eb64f26bdd00b4beb44c896 is 175, key is test_row_0/A:col10/1731310203136/Put/seqid=0 2024-11-11T07:30:03,800 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/5188f9e66a2d4bd1a6650e387863a122 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/5188f9e66a2d4bd1a6650e387863a122 2024-11-11T07:30:03,806 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/a732fe8d1b4040e293705d2bc233734a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/a732fe8d1b4040e293705d2bc233734a 2024-11-11T07:30:03,820 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/A of 18312f0c839a758e8897bb39c81d93ff into 5188f9e66a2d4bd1a6650e387863a122(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:03,820 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:03,820 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/A, priority=13, startTime=1731310203137; duration=0sec 2024-11-11T07:30:03,821 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:03,821 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:A 2024-11-11T07:30:03,821 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:03,824 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:03,824 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/C is initiating minor compaction (all files) 2024-11-11T07:30:03,824 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/C in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:03,824 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/5691605df52c469eb114daa05a752408, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/ef21f379dd334c23beb2300102a3342c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/879e20cbe2e84fb4824810bed58e1ed5] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=35.9 K 2024-11-11T07:30:03,825 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5691605df52c469eb114daa05a752408, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731310199569 2024-11-11T07:30:03,826 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef21f379dd334c23beb2300102a3342c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731310200780 2024-11-11T07:30:03,826 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 879e20cbe2e84fb4824810bed58e1ed5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731310201191 2024-11-11T07:30:03,859 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/B of 18312f0c839a758e8897bb39c81d93ff into a732fe8d1b4040e293705d2bc233734a(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:03,859 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:03,859 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/B, priority=13, startTime=1731310203143; duration=0sec 2024-11-11T07:30:03,859 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:03,859 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:B 2024-11-11T07:30:03,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742010_1186 (size=39749) 2024-11-11T07:30:03,878 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#C#compaction#156 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:03,878 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/c9d5a9853da04a39872b1ca68e930add is 50, key is test_row_0/C:col10/1731310201197/Put/seqid=0 2024-11-11T07:30:03,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:03,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:03,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:03,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742011_1187 (size=12595) 2024-11-11T07:30:03,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:03,935 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/c9d5a9853da04a39872b1ca68e930add as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/c9d5a9853da04a39872b1ca68e930add 2024-11-11T07:30:03,955 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/C of 18312f0c839a758e8897bb39c81d93ff into c9d5a9853da04a39872b1ca68e930add(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:03,955 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:03,955 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/C, priority=13, startTime=1731310203143; duration=0sec 2024-11-11T07:30:03,955 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:03,955 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:C 2024-11-11T07:30:04,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:04,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310264004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:04,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:04,075 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:04,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:04,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:04,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:04,076 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:04,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,275 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=205, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/4025bc6b0eb64f26bdd00b4beb44c896 2024-11-11T07:30:04,304 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/b909e28f74b34ea595d7855bd40f5c0f is 50, key is test_row_0/B:col10/1731310203136/Put/seqid=0 2024-11-11T07:30:04,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742012_1188 (size=12151) 2024-11-11T07:30:04,370 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/b909e28f74b34ea595d7855bd40f5c0f 2024-11-11T07:30:04,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:04,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:04,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:04,394 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/76d2172f2c684083aee0738fdd16a87a is 50, key is test_row_0/C:col10/1731310203136/Put/seqid=0 2024-11-11T07:30:04,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742013_1189 (size=12151) 2024-11-11T07:30:04,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:04,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310264539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:04,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:04,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:04,549 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:04,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:04,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:04,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:04,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:04,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:04,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310264639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:04,655 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_0' in 18312f0c839a758e8897bb39c81d93ff of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:04,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:04,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:04,655 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:04,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:04,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310264654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:04,656 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_0' in 18312f0c839a758e8897bb39c81d93ff of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:04,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:04,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:04,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:04,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:04,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310264647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:04,663 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_1' in 18312f0c839a758e8897bb39c81d93ff of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:04,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:04,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:04,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:04,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:04,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310264685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:04,689 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_2' in 18312f0c839a758e8897bb39c81d93ff of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:04,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:04,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:04,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:04,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:04,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:04,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:04,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:04,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,883 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:04,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:04,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:04,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:04,887 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:04,926 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/76d2172f2c684083aee0738fdd16a87a 2024-11-11T07:30:04,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/4025bc6b0eb64f26bdd00b4beb44c896 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/4025bc6b0eb64f26bdd00b4beb44c896 2024-11-11T07:30:04,946 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/4025bc6b0eb64f26bdd00b4beb44c896, entries=200, sequenceid=205, filesize=38.8 K 2024-11-11T07:30:04,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/b909e28f74b34ea595d7855bd40f5c0f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/b909e28f74b34ea595d7855bd40f5c0f 2024-11-11T07:30:04,955 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/b909e28f74b34ea595d7855bd40f5c0f, entries=150, sequenceid=205, filesize=11.9 K 2024-11-11T07:30:04,956 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/76d2172f2c684083aee0738fdd16a87a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/76d2172f2c684083aee0738fdd16a87a 2024-11-11T07:30:04,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/76d2172f2c684083aee0738fdd16a87a, entries=150, sequenceid=205, filesize=11.9 K 2024-11-11T07:30:05,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 18312f0c839a758e8897bb39c81d93ff in 1838ms, sequenceid=205, compaction requested=false 2024-11-11T07:30:05,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:05,049 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-11T07:30:05,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:05,049 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T07:30:05,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:30:05,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:05,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:30:05,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:05,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:30:05,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:05,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f53dca361d8e4f5287173c3a50f3adb9_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310203323/Put/seqid=0 2024-11-11T07:30:05,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742014_1190 (size=12304) 2024-11-11T07:30:05,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T07:30:05,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:05,524 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f53dca361d8e4f5287173c3a50f3adb9_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f53dca361d8e4f5287173c3a50f3adb9_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:05,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/912472bd580a4a10b940865b8f9989ae, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:05,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/912472bd580a4a10b940865b8f9989ae is 175, key is test_row_0/A:col10/1731310203323/Put/seqid=0 2024-11-11T07:30:05,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:05,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:05,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742015_1191 (size=31105) 2024-11-11T07:30:05,606 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=226, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/912472bd580a4a10b940865b8f9989ae 2024-11-11T07:30:05,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310265686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:05,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/273fe1175a8044208d95c5e48aed78a6 is 50, key is test_row_0/B:col10/1731310203323/Put/seqid=0 2024-11-11T07:30:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:05,697 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:05,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742016_1192 (size=12151) 2024-11-11T07:30:05,747 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/273fe1175a8044208d95c5e48aed78a6 2024-11-11T07:30:05,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/9fb78e3d7ea0473ba7b321a92839a3c9 is 50, key is test_row_0/C:col10/1731310203323/Put/seqid=0 2024-11-11T07:30:05,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:05,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310265806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:05,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:05,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742017_1193 (size=12151) 2024-11-11T07:30:05,838 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=226 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/9fb78e3d7ea0473ba7b321a92839a3c9 2024-11-11T07:30:05,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/912472bd580a4a10b940865b8f9989ae as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/912472bd580a4a10b940865b8f9989ae 2024-11-11T07:30:05,883 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/912472bd580a4a10b940865b8f9989ae, entries=150, sequenceid=226, filesize=30.4 K 2024-11-11T07:30:05,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/273fe1175a8044208d95c5e48aed78a6 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/273fe1175a8044208d95c5e48aed78a6 2024-11-11T07:30:05,909 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/273fe1175a8044208d95c5e48aed78a6, entries=150, sequenceid=226, filesize=11.9 K 2024-11-11T07:30:05,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/9fb78e3d7ea0473ba7b321a92839a3c9 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/9fb78e3d7ea0473ba7b321a92839a3c9 2024-11-11T07:30:05,917 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/9fb78e3d7ea0473ba7b321a92839a3c9, entries=150, sequenceid=226, filesize=11.9 K 2024-11-11T07:30:05,919 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 18312f0c839a758e8897bb39c81d93ff in 869ms, sequenceid=226, compaction requested=true 2024-11-11T07:30:05,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:05,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:05,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-11T07:30:05,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-11T07:30:05,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-11T07:30:05,923 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 4.8280 sec 2024-11-11T07:30:05,925 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 4.8410 sec 2024-11-11T07:30:06,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:06,024 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T07:30:06,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:30:06,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:06,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:30:06,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:06,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:30:06,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:06,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111ebc296e6213416abc0e510e1c4e0709_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310205665/Put/seqid=0 2024-11-11T07:30:06,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742018_1194 (size=14794) 2024-11-11T07:30:06,112 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:06,116 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411111ebc296e6213416abc0e510e1c4e0709_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111ebc296e6213416abc0e510e1c4e0709_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:06,117 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/a88bf3e67cb541b7bf51498d79245c22, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:06,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/a88bf3e67cb541b7bf51498d79245c22 is 175, key is test_row_0/A:col10/1731310205665/Put/seqid=0 2024-11-11T07:30:06,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742019_1195 (size=39749) 2024-11-11T07:30:06,180 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=241, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/a88bf3e67cb541b7bf51498d79245c22 2024-11-11T07:30:06,200 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/55775e4eaf9944458c3bc5d10e74fa16 is 50, key is test_row_0/B:col10/1731310205665/Put/seqid=0 2024-11-11T07:30:06,217 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:06,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310266212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:06,218 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:06,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742020_1196 (size=12151) 2024-11-11T07:30:06,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/55775e4eaf9944458c3bc5d10e74fa16 2024-11-11T07:30:06,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/93ef7caa93c243e7845934f1f40ea181 is 50, key is test_row_0/C:col10/1731310205665/Put/seqid=0 2024-11-11T07:30:06,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742021_1197 (size=12151) 2024-11-11T07:30:06,345 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:06,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310266326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:06,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:06,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:06,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310266554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:06,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:06,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:06,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60848 deadline: 1731310266665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:06,671 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_0' in 18312f0c839a758e8897bb39c81d93ff of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4215 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:06,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:06,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:06,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:06,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:06,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60812 deadline: 1731310266666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:06,679 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_0' in 18312f0c839a758e8897bb39c81d93ff of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:06,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:06,689 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:06,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60866 deadline: 1731310266674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:06,690 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_1' in 18312f0c839a758e8897bb39c81d93ff of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4241 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:06,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:06,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:06,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:06,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:06,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60806 deadline: 1731310266707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:06,714 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_2' in 18312f0c839a758e8897bb39c81d93ff of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4272 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:06,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:06,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:06,715 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:06,788 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=241 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/93ef7caa93c243e7845934f1f40ea181 2024-11-11T07:30:06,794 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/a88bf3e67cb541b7bf51498d79245c22 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/a88bf3e67cb541b7bf51498d79245c22 2024-11-11T07:30:06,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/a88bf3e67cb541b7bf51498d79245c22, entries=200, sequenceid=241, filesize=38.8 K 2024-11-11T07:30:06,800 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/55775e4eaf9944458c3bc5d10e74fa16 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/55775e4eaf9944458c3bc5d10e74fa16 2024-11-11T07:30:06,810 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/55775e4eaf9944458c3bc5d10e74fa16, entries=150, sequenceid=241, filesize=11.9 K 2024-11-11T07:30:06,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/93ef7caa93c243e7845934f1f40ea181 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/93ef7caa93c243e7845934f1f40ea181 2024-11-11T07:30:06,817 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/93ef7caa93c243e7845934f1f40ea181, entries=150, sequenceid=241, filesize=11.9 K 2024-11-11T07:30:06,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 18312f0c839a758e8897bb39c81d93ff in 794ms, sequenceid=241, compaction requested=true 2024-11-11T07:30:06,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:06,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:06,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:06,818 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:06,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:06,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:06,818 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:06,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:06,818 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:06,820 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142152 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:06,820 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:06,820 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/B is initiating minor compaction (all files) 2024-11-11T07:30:06,820 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/A is initiating minor compaction (all files) 2024-11-11T07:30:06,820 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/B in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:06,820 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/A in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:06,820 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/a732fe8d1b4040e293705d2bc233734a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/b909e28f74b34ea595d7855bd40f5c0f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/273fe1175a8044208d95c5e48aed78a6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/55775e4eaf9944458c3bc5d10e74fa16] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=47.9 K 2024-11-11T07:30:06,820 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/5188f9e66a2d4bd1a6650e387863a122, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/4025bc6b0eb64f26bdd00b4beb44c896, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/912472bd580a4a10b940865b8f9989ae, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/a88bf3e67cb541b7bf51498d79245c22] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=138.8 K 2024-11-11T07:30:06,820 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:06,820 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/5188f9e66a2d4bd1a6650e387863a122, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/4025bc6b0eb64f26bdd00b4beb44c896, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/912472bd580a4a10b940865b8f9989ae, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/a88bf3e67cb541b7bf51498d79245c22] 2024-11-11T07:30:06,821 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting a732fe8d1b4040e293705d2bc233734a, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731310201191 2024-11-11T07:30:06,821 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5188f9e66a2d4bd1a6650e387863a122, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731310201191 2024-11-11T07:30:06,821 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4025bc6b0eb64f26bdd00b4beb44c896, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731310202424 2024-11-11T07:30:06,821 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting b909e28f74b34ea595d7855bd40f5c0f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731310202448 2024-11-11T07:30:06,822 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 273fe1175a8044208d95c5e48aed78a6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731310203319 2024-11-11T07:30:06,822 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 912472bd580a4a10b940865b8f9989ae, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731310203319 2024-11-11T07:30:06,822 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 55775e4eaf9944458c3bc5d10e74fa16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731310205665 2024-11-11T07:30:06,822 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting a88bf3e67cb541b7bf51498d79245c22, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731310205657 2024-11-11T07:30:06,839 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#B#compaction#165 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:06,839 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/1749c85467b14b699cbc012a14aedf9f is 50, key is test_row_0/B:col10/1731310205665/Put/seqid=0 2024-11-11T07:30:06,849 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:06,869 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111631a2da792a14885815dcce9bd2b7a15_18312f0c839a758e8897bb39c81d93ff store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:06,872 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111631a2da792a14885815dcce9bd2b7a15_18312f0c839a758e8897bb39c81d93ff, store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:06,872 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111631a2da792a14885815dcce9bd2b7a15_18312f0c839a758e8897bb39c81d93ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:06,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:06,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T07:30:06,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:30:06,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:06,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:30:06,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:06,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:30:06,876 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:06,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114d403f5de52e4f5e812dce1d08155f9e_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310206210/Put/seqid=0 2024-11-11T07:30:06,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742022_1198 (size=12731) 2024-11-11T07:30:06,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742023_1199 (size=4469) 2024-11-11T07:30:06,933 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/1749c85467b14b699cbc012a14aedf9f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/1749c85467b14b699cbc012a14aedf9f 2024-11-11T07:30:06,941 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/B of 18312f0c839a758e8897bb39c81d93ff into 1749c85467b14b699cbc012a14aedf9f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:06,941 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:06,941 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/B, priority=12, startTime=1731310206818; duration=0sec 2024-11-11T07:30:06,941 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#A#compaction#166 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:06,942 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/d4b177b2b4d041789b182d242624d1eb is 175, key is test_row_0/A:col10/1731310205665/Put/seqid=0 2024-11-11T07:30:06,942 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:06,942 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:B 2024-11-11T07:30:06,942 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:06,946 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:06,946 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/C is initiating minor compaction (all files) 2024-11-11T07:30:06,946 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/C in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:06,946 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/c9d5a9853da04a39872b1ca68e930add, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/76d2172f2c684083aee0738fdd16a87a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/9fb78e3d7ea0473ba7b321a92839a3c9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/93ef7caa93c243e7845934f1f40ea181] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=47.9 K 2024-11-11T07:30:06,947 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting c9d5a9853da04a39872b1ca68e930add, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731310201191 2024-11-11T07:30:06,947 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 76d2172f2c684083aee0738fdd16a87a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731310202448 2024-11-11T07:30:06,948 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 9fb78e3d7ea0473ba7b321a92839a3c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=226, earliestPutTs=1731310203319 2024-11-11T07:30:06,951 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 93ef7caa93c243e7845934f1f40ea181, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731310205665 2024-11-11T07:30:06,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742024_1200 (size=14894) 2024-11-11T07:30:06,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742025_1201 (size=31685) 2024-11-11T07:30:06,991 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#C#compaction#168 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:06,992 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/f58dad78a6ac4757810973d7cb2d1565 is 50, key is test_row_0/C:col10/1731310205665/Put/seqid=0 2024-11-11T07:30:07,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:07,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742026_1202 (size=12731) 2024-11-11T07:30:07,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310267012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:07,024 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:07,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:07,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310267139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:07,143 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:07,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:07,355 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:07,366 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310267359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:07,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:07,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:07,367 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:07,372 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114d403f5de52e4f5e812dce1d08155f9e_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114d403f5de52e4f5e812dce1d08155f9e_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:07,386 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/6134fad50e304bd790780482abd89dbc, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:07,387 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/6134fad50e304bd790780482abd89dbc is 175, key is test_row_0/A:col10/1731310206210/Put/seqid=0 2024-11-11T07:30:07,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742027_1203 (size=39849) 2024-11-11T07:30:07,431 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/d4b177b2b4d041789b182d242624d1eb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/d4b177b2b4d041789b182d242624d1eb 2024-11-11T07:30:07,434 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=259, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/6134fad50e304bd790780482abd89dbc 2024-11-11T07:30:07,439 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/f58dad78a6ac4757810973d7cb2d1565 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/f58dad78a6ac4757810973d7cb2d1565 2024-11-11T07:30:07,448 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/A of 18312f0c839a758e8897bb39c81d93ff into d4b177b2b4d041789b182d242624d1eb(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:07,448 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:07,448 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/A, priority=12, startTime=1731310206818; duration=0sec 2024-11-11T07:30:07,448 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:07,448 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:A 2024-11-11T07:30:07,452 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/C of 18312f0c839a758e8897bb39c81d93ff into f58dad78a6ac4757810973d7cb2d1565(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:07,452 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:07,452 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/C, priority=12, startTime=1731310206818; duration=0sec 2024-11-11T07:30:07,452 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:07,452 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:C 2024-11-11T07:30:07,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/762ac753f73d4cdea41388f78d9f42a0 is 50, key is test_row_0/B:col10/1731310206210/Put/seqid=0 2024-11-11T07:30:07,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742028_1204 (size=12251) 2024-11-11T07:30:07,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/762ac753f73d4cdea41388f78d9f42a0 2024-11-11T07:30:07,548 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/d5f666f7010e464e90ac898ab0270899 is 50, key is test_row_0/C:col10/1731310206210/Put/seqid=0 2024-11-11T07:30:07,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742029_1205 (size=12251) 2024-11-11T07:30:07,683 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:07,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310267679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:07,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:07,993 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/d5f666f7010e464e90ac898ab0270899 2024-11-11T07:30:08,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/6134fad50e304bd790780482abd89dbc as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/6134fad50e304bd790780482abd89dbc 2024-11-11T07:30:08,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/6134fad50e304bd790780482abd89dbc, entries=200, sequenceid=259, filesize=38.9 K 2024-11-11T07:30:08,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/762ac753f73d4cdea41388f78d9f42a0 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/762ac753f73d4cdea41388f78d9f42a0 2024-11-11T07:30:08,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/762ac753f73d4cdea41388f78d9f42a0, entries=150, sequenceid=259, filesize=12.0 K 2024-11-11T07:30:08,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/d5f666f7010e464e90ac898ab0270899 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/d5f666f7010e464e90ac898ab0270899 2024-11-11T07:30:08,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/d5f666f7010e464e90ac898ab0270899, entries=150, sequenceid=259, filesize=12.0 K 2024-11-11T07:30:08,030 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 18312f0c839a758e8897bb39c81d93ff in 1154ms, sequenceid=259, compaction requested=false 2024-11-11T07:30:08,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:08,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T07:30:08,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:30:08,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:08,199 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:30:08,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:08,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:08,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:30:08,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:08,222 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118e03be549a044c29b2f60ec173e0499f_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310208196/Put/seqid=0 2024-11-11T07:30:08,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742030_1206 (size=14994) 2024-11-11T07:30:08,279 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:08,318 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118e03be549a044c29b2f60ec173e0499f_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118e03be549a044c29b2f60ec173e0499f_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:08,334 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/44644f782ba14ff88bfe4f4c7342a162, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:08,335 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/44644f782ba14ff88bfe4f4c7342a162 is 175, key is test_row_0/A:col10/1731310208196/Put/seqid=0 2024-11-11T07:30:08,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742031_1207 (size=39949) 2024-11-11T07:30:08,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:08,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310268396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:08,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:08,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:08,399 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:08,510 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310268505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:08,511 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:08,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310268719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:08,727 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:08,790 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=277, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/44644f782ba14ff88bfe4f4c7342a162 2024-11-11T07:30:08,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/6e1e6061a0d54201a567974c5a1d36af is 50, key is test_row_0/B:col10/1731310208196/Put/seqid=0 2024-11-11T07:30:08,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742032_1208 (size=12301) 2024-11-11T07:30:08,904 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/6e1e6061a0d54201a567974c5a1d36af 2024-11-11T07:30:08,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/87abbd0046d54d80b7218435a82dcd67 is 50, key is test_row_0/C:col10/1731310208196/Put/seqid=0 2024-11-11T07:30:09,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742033_1209 (size=12301) 2024-11-11T07:30:09,039 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:09,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310269035, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:09,040 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:09,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-11T07:30:09,339 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:09,346 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:09,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-11T07:30:09,352 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:09,353 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:09,353 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T07:30:09,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/87abbd0046d54d80b7218435a82dcd67 2024-11-11T07:30:09,412 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/44644f782ba14ff88bfe4f4c7342a162 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/44644f782ba14ff88bfe4f4c7342a162 2024-11-11T07:30:09,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/44644f782ba14ff88bfe4f4c7342a162, entries=200, sequenceid=277, filesize=39.0 K 2024-11-11T07:30:09,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/6e1e6061a0d54201a567974c5a1d36af as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/6e1e6061a0d54201a567974c5a1d36af 2024-11-11T07:30:09,442 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/6e1e6061a0d54201a567974c5a1d36af, entries=150, sequenceid=277, filesize=12.0 K 2024-11-11T07:30:09,443 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/87abbd0046d54d80b7218435a82dcd67 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/87abbd0046d54d80b7218435a82dcd67 2024-11-11T07:30:09,452 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/87abbd0046d54d80b7218435a82dcd67, entries=150, sequenceid=277, filesize=12.0 K 2024-11-11T07:30:09,456 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 18312f0c839a758e8897bb39c81d93ff in 1258ms, sequenceid=277, compaction requested=true 2024-11-11T07:30:09,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:09,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:09,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:09,456 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:09,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:09,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:09,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 18312f0c839a758e8897bb39c81d93ff:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:09,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:30:09,456 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:09,463 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:09,463 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/B is initiating minor compaction (all files) 2024-11-11T07:30:09,463 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/B in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:09,463 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/1749c85467b14b699cbc012a14aedf9f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/762ac753f73d4cdea41388f78d9f42a0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/6e1e6061a0d54201a567974c5a1d36af] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=36.4 K 2024-11-11T07:30:09,463 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111483 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:09,463 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/A is initiating minor compaction (all files) 2024-11-11T07:30:09,463 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/A in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:09,464 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/d4b177b2b4d041789b182d242624d1eb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/6134fad50e304bd790780482abd89dbc, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/44644f782ba14ff88bfe4f4c7342a162] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=108.9 K 2024-11-11T07:30:09,464 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:09,464 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/d4b177b2b4d041789b182d242624d1eb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/6134fad50e304bd790780482abd89dbc, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/44644f782ba14ff88bfe4f4c7342a162] 2024-11-11T07:30:09,466 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4b177b2b4d041789b182d242624d1eb, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731310205665 2024-11-11T07:30:09,467 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 1749c85467b14b699cbc012a14aedf9f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731310205665 2024-11-11T07:30:09,467 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6134fad50e304bd790780482abd89dbc, keycount=200, bloomtype=ROW, size=38.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731310206173 2024-11-11T07:30:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T07:30:09,471 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 762ac753f73d4cdea41388f78d9f42a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731310206183 2024-11-11T07:30:09,471 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 44644f782ba14ff88bfe4f4c7342a162, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731310206928 2024-11-11T07:30:09,475 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 6e1e6061a0d54201a567974c5a1d36af, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731310206928 2024-11-11T07:30:09,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-11T07:30:09,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:09,515 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T07:30:09,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:30:09,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:09,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:30:09,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:09,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:30:09,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:09,521 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#B#compaction#174 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:09,521 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/3776e761da664d89b1aa94a56ffff728 is 50, key is test_row_0/B:col10/1731310208196/Put/seqid=0 2024-11-11T07:30:09,527 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:09,547 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411110c4c00044ab5433f9639a272388f5d02_18312f0c839a758e8897bb39c81d93ff store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:09,549 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411110c4c00044ab5433f9639a272388f5d02_18312f0c839a758e8897bb39c81d93ff, store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:09,550 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110c4c00044ab5433f9639a272388f5d02_18312f0c839a758e8897bb39c81d93ff because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:09,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:09,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112a6fd4e208c3486abb528f36d2bf61c7_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310208386/Put/seqid=0 2024-11-11T07:30:09,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742034_1210 (size=12983) 2024-11-11T07:30:09,596 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/3776e761da664d89b1aa94a56ffff728 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/3776e761da664d89b1aa94a56ffff728 2024-11-11T07:30:09,614 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/B of 18312f0c839a758e8897bb39c81d93ff into 3776e761da664d89b1aa94a56ffff728(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:09,614 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:09,614 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/B, priority=13, startTime=1731310209456; duration=0sec 2024-11-11T07:30:09,614 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:09,614 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:B 2024-11-11T07:30:09,615 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:09,618 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37283 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:09,618 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 18312f0c839a758e8897bb39c81d93ff/C is initiating minor compaction (all files) 2024-11-11T07:30:09,618 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 18312f0c839a758e8897bb39c81d93ff/C in TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:09,618 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/f58dad78a6ac4757810973d7cb2d1565, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/d5f666f7010e464e90ac898ab0270899, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/87abbd0046d54d80b7218435a82dcd67] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp, totalSize=36.4 K 2024-11-11T07:30:09,619 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f58dad78a6ac4757810973d7cb2d1565, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=241, earliestPutTs=1731310205665 2024-11-11T07:30:09,619 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting d5f666f7010e464e90ac898ab0270899, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731310206183 2024-11-11T07:30:09,620 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 87abbd0046d54d80b7218435a82dcd67, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731310206928 2024-11-11T07:30:09,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742035_1211 (size=4469) 2024-11-11T07:30:09,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742036_1212 (size=12454) 2024-11-11T07:30:09,631 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#A#compaction#175 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:09,631 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/cd8e52593b0f4b5bb7e613007d98fc88 is 175, key is test_row_0/A:col10/1731310208196/Put/seqid=0 2024-11-11T07:30:09,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:09,643 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 18312f0c839a758e8897bb39c81d93ff#C#compaction#177 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:09,644 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/d7a2c92c17e84bf585b26d6c39c3cdb3 is 50, key is test_row_0/C:col10/1731310208196/Put/seqid=0 2024-11-11T07:30:09,671 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112a6fd4e208c3486abb528f36d2bf61c7_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112a6fd4e208c3486abb528f36d2bf61c7_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T07:30:09,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/b8d80ca6d3be4e0cbf37f1153745d44a, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:09,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/b8d80ca6d3be4e0cbf37f1153745d44a is 175, key is test_row_0/A:col10/1731310208386/Put/seqid=0 2024-11-11T07:30:09,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742037_1213 (size=31937) 2024-11-11T07:30:09,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310269682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:09,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:09,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742038_1214 (size=12983) 2024-11-11T07:30:09,739 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/d7a2c92c17e84bf585b26d6c39c3cdb3 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/d7a2c92c17e84bf585b26d6c39c3cdb3 2024-11-11T07:30:09,744 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/C of 18312f0c839a758e8897bb39c81d93ff into d7a2c92c17e84bf585b26d6c39c3cdb3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:09,744 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:09,745 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/C, priority=13, startTime=1731310209456; duration=0sec 2024-11-11T07:30:09,745 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:09,745 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:C 2024-11-11T07:30:09,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742039_1215 (size=31255) 2024-11-11T07:30:09,760 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=294, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/b8d80ca6d3be4e0cbf37f1153745d44a 2024-11-11T07:30:09,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/a165b871a7b042eab0da8dd8fee329fe is 50, key is test_row_0/B:col10/1731310208386/Put/seqid=0 2024-11-11T07:30:09,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:09,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310269808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:09,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:09,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:09,813 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:09,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742040_1216 (size=12301) 2024-11-11T07:30:09,827 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/a165b871a7b042eab0da8dd8fee329fe 2024-11-11T07:30:09,860 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/5d5220e2b3504c8d9319feb3481e43ec is 50, key is test_row_0/C:col10/1731310208386/Put/seqid=0 2024-11-11T07:30:09,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742041_1217 (size=12301) 2024-11-11T07:30:09,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T07:30:10,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:10,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310270018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:10,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:10,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:10,021 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:10,101 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/cd8e52593b0f4b5bb7e613007d98fc88 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/cd8e52593b0f4b5bb7e613007d98fc88 2024-11-11T07:30:10,114 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 18312f0c839a758e8897bb39c81d93ff/A of 18312f0c839a758e8897bb39c81d93ff into cd8e52593b0f4b5bb7e613007d98fc88(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:10,114 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:10,114 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., storeName=18312f0c839a758e8897bb39c81d93ff/A, priority=13, startTime=1731310209456; duration=0sec 2024-11-11T07:30:10,114 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:10,114 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 18312f0c839a758e8897bb39c81d93ff:A 2024-11-11T07:30:10,322 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=294 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/5d5220e2b3504c8d9319feb3481e43ec 2024-11-11T07:30:10,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:10,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:60860 deadline: 1731310270327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:10,333 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:10,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/b8d80ca6d3be4e0cbf37f1153745d44a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/b8d80ca6d3be4e0cbf37f1153745d44a 2024-11-11T07:30:10,351 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/b8d80ca6d3be4e0cbf37f1153745d44a, entries=150, sequenceid=294, filesize=30.5 K 2024-11-11T07:30:10,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/a165b871a7b042eab0da8dd8fee329fe as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/a165b871a7b042eab0da8dd8fee329fe 2024-11-11T07:30:10,399 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/a165b871a7b042eab0da8dd8fee329fe, entries=150, sequenceid=294, filesize=12.0 K 2024-11-11T07:30:10,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/5d5220e2b3504c8d9319feb3481e43ec as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/5d5220e2b3504c8d9319feb3481e43ec 2024-11-11T07:30:10,451 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/5d5220e2b3504c8d9319feb3481e43ec, entries=150, sequenceid=294, filesize=12.0 K 2024-11-11T07:30:10,458 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 18312f0c839a758e8897bb39c81d93ff in 943ms, sequenceid=294, compaction requested=false 2024-11-11T07:30:10,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:10,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:10,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-11T07:30:10,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-11T07:30:10,478 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-11T07:30:10,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1120 sec 2024-11-11T07:30:10,481 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.1320 sec 2024-11-11T07:30:10,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-11T07:30:10,514 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:10,515 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:10,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-11T07:30:10,517 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:10,518 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:10,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:10,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T07:30:10,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T07:30:10,657 INFO [Thread-659 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-659. 2024-11-11T07:30:10,657 DEBUG [Thread-659 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:10,657 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,657 DEBUG [Thread-659 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,658 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:10,660 INFO [Thread-665 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-665. 2024-11-11T07:30:10,660 DEBUG [Thread-665 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:10,660 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,660 DEBUG [Thread-665 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,662 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:10,662 INFO [Thread-663 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-663. 2024-11-11T07:30:10,663 DEBUG [Thread-663 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:10,663 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,663 DEBUG [Thread-663 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,663 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:10,664 INFO [Thread-661 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-661. 2024-11-11T07:30:10,664 DEBUG [Thread-661 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:10,664 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,664 DEBUG [Thread-661 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,664 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:10,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-11T07:30:10,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:10,672 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T07:30:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:30:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:30:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:30:10,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:10,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ccb596bc50044dd4a38a150b42e1924d_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310209668/Put/seqid=0 2024-11-11T07:30:10,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742042_1218 (size=12454) 2024-11-11T07:30:10,706 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. as already flushing 2024-11-11T07:30:10,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:10,706 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-11T07:30:10,706 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:10,706 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,706 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-11T07:30:10,706 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,706 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:10,706 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,706 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,706 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:10,707 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:10,716 INFO [Thread-656 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-656. 2024-11-11T07:30:10,716 DEBUG [Thread-656 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:10,716 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,716 DEBUG [Thread-656 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:10,736 INFO [Thread-652 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-652. 2024-11-11T07:30:10,737 DEBUG [Thread-652 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:10,737 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,737 DEBUG [Thread-652 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,737 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:10,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T07:30:10,837 INFO [Thread-654 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-654. 2024-11-11T07:30:10,837 DEBUG [Thread-654 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:10,837 DEBUG [Thread-654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,837 DEBUG [Thread-654 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:10,837 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:11,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:11,093 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ccb596bc50044dd4a38a150b42e1924d_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ccb596bc50044dd4a38a150b42e1924d_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:11,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/02eb9ba45df0467cb7341904616c17b5, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:11,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/02eb9ba45df0467cb7341904616c17b5 is 175, key is test_row_0/A:col10/1731310209668/Put/seqid=0 2024-11-11T07:30:11,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742043_1219 (size=31255) 2024-11-11T07:30:11,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T07:30:11,505 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=312, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/02eb9ba45df0467cb7341904616c17b5 2024-11-11T07:30:11,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/c97d85c8c80344279c8ac71e6f6221d6 is 50, key is test_row_0/B:col10/1731310209668/Put/seqid=0 2024-11-11T07:30:11,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742044_1220 (size=12301) 2024-11-11T07:30:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T07:30:11,919 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/c97d85c8c80344279c8ac71e6f6221d6 2024-11-11T07:30:11,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/30224cabf8874257a43c185c697cb2fe is 50, key is test_row_0/C:col10/1731310209668/Put/seqid=0 2024-11-11T07:30:11,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742045_1221 (size=12301) 2024-11-11T07:30:11,937 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/30224cabf8874257a43c185c697cb2fe 2024-11-11T07:30:11,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/02eb9ba45df0467cb7341904616c17b5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/02eb9ba45df0467cb7341904616c17b5 2024-11-11T07:30:11,948 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/02eb9ba45df0467cb7341904616c17b5, entries=150, sequenceid=312, filesize=30.5 K 2024-11-11T07:30:11,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/c97d85c8c80344279c8ac71e6f6221d6 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/c97d85c8c80344279c8ac71e6f6221d6 2024-11-11T07:30:11,955 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/c97d85c8c80344279c8ac71e6f6221d6, entries=150, sequenceid=312, filesize=12.0 K 2024-11-11T07:30:11,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/30224cabf8874257a43c185c697cb2fe as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/30224cabf8874257a43c185c697cb2fe 2024-11-11T07:30:11,962 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/30224cabf8874257a43c185c697cb2fe, entries=150, sequenceid=312, filesize=12.0 K 2024-11-11T07:30:11,963 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=33.54 KB/34350 for 18312f0c839a758e8897bb39c81d93ff in 1292ms, sequenceid=312, compaction requested=true 2024-11-11T07:30:11,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 18312f0c839a758e8897bb39c81d93ff: 2024-11-11T07:30:11,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:11,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-11T07:30:11,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-11T07:30:11,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-11T07:30:11,967 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4470 sec 2024-11-11T07:30:11,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.4530 sec 2024-11-11T07:30:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-11T07:30:12,665 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:12,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T07:30:12,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 35 2024-11-11T07:30:12,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-11T07:30:12,665 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-11T07:30:12,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-11T07:30:12,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-11T07:30:12,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T07:30:12,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1708 2024-11-11T07:30:12,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1684 2024-11-11T07:30:12,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T07:30:12,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 758 2024-11-11T07:30:12,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2274 rows 2024-11-11T07:30:12,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 785 2024-11-11T07:30:12,666 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2352 rows 2024-11-11T07:30:12,666 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T07:30:12,666 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:30:12,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:12,666 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:12,666 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:12,667 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T07:30:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T07:30:12,670 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310212670"}]},"ts":"1731310212670"} 2024-11-11T07:30:12,672 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T07:30:12,672 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T07:30:12,673 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:30:12,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18312f0c839a758e8897bb39c81d93ff, UNASSIGN}] 2024-11-11T07:30:12,675 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18312f0c839a758e8897bb39c81d93ff, UNASSIGN 2024-11-11T07:30:12,676 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=18312f0c839a758e8897bb39c81d93ff, regionState=CLOSING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:12,677 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=54, ppid=53, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=18312f0c839a758e8897bb39c81d93ff, UNASSIGN because future has completed 2024-11-11T07:30:12,677 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:30:12,677 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:30:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T07:30:12,830 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(122): Close 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:12,830 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:30:12,830 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1722): Closing 18312f0c839a758e8897bb39c81d93ff, disabling compactions & flushes 2024-11-11T07:30:12,830 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:12,830 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:12,830 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. after waiting 0 ms 2024-11-11T07:30:12,831 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:12,831 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(2902): Flushing 18312f0c839a758e8897bb39c81d93ff 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T07:30:12,831 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=A 2024-11-11T07:30:12,831 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:12,831 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=B 2024-11-11T07:30:12,831 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:12,831 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 18312f0c839a758e8897bb39c81d93ff, store=C 2024-11-11T07:30:12,831 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:12,838 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111497e3ef2e2a84369ac121c8cf7b89ec7_18312f0c839a758e8897bb39c81d93ff is 50, key is test_row_0/A:col10/1731310210715/Put/seqid=0 2024-11-11T07:30:12,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742046_1222 (size=12454) 2024-11-11T07:30:12,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T07:30:13,242 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:13,247 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111497e3ef2e2a84369ac121c8cf7b89ec7_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111497e3ef2e2a84369ac121c8cf7b89ec7_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:13,248 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/e248aa534ee749b48c90356dfd5314f6, store: [table=TestAcidGuarantees family=A region=18312f0c839a758e8897bb39c81d93ff] 2024-11-11T07:30:13,248 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/e248aa534ee749b48c90356dfd5314f6 is 175, key is test_row_0/A:col10/1731310210715/Put/seqid=0 2024-11-11T07:30:13,255 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742047_1223 (size=31255) 2024-11-11T07:30:13,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T07:30:13,334 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T07:30:13,655 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/e248aa534ee749b48c90356dfd5314f6 2024-11-11T07:30:13,663 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/e1d9142021794fa18c558d3318960962 is 50, key is test_row_0/B:col10/1731310210715/Put/seqid=0 2024-11-11T07:30:13,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742048_1224 (size=12301) 2024-11-11T07:30:13,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T07:30:14,068 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/e1d9142021794fa18c558d3318960962 2024-11-11T07:30:14,075 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/b02d61ec4d6941c9ba9aec233cc62896 is 50, key is test_row_0/C:col10/1731310210715/Put/seqid=0 2024-11-11T07:30:14,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742049_1225 (size=12301) 2024-11-11T07:30:14,479 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/b02d61ec4d6941c9ba9aec233cc62896 2024-11-11T07:30:14,485 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/A/e248aa534ee749b48c90356dfd5314f6 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/e248aa534ee749b48c90356dfd5314f6 2024-11-11T07:30:14,490 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/e248aa534ee749b48c90356dfd5314f6, entries=150, sequenceid=320, filesize=30.5 K 2024-11-11T07:30:14,492 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/B/e1d9142021794fa18c558d3318960962 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/e1d9142021794fa18c558d3318960962 2024-11-11T07:30:14,496 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/e1d9142021794fa18c558d3318960962, entries=150, sequenceid=320, filesize=12.0 K 2024-11-11T07:30:14,497 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/.tmp/C/b02d61ec4d6941c9ba9aec233cc62896 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/b02d61ec4d6941c9ba9aec233cc62896 2024-11-11T07:30:14,501 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/b02d61ec4d6941c9ba9aec233cc62896, entries=150, sequenceid=320, filesize=12.0 K 2024-11-11T07:30:14,502 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 18312f0c839a758e8897bb39c81d93ff in 1671ms, sequenceid=320, compaction requested=true 2024-11-11T07:30:14,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/82d8c2e6825b414d93b6edd0c287638d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/1ebd3ac1149a4e13887f440e2ae79e92, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/f9d9421655cc446f920930dc2adb32cf, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/b52402183a184fce929716b91a3ed5e9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/85db1f5653cc46a78e32176521a5ae02, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/03d20c5fc85547698aaaf983cbee57ed, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/6e417d57a4274f4daf0944e6b0ac35dd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/e5d95791a6334f38a96b2a163c6686a1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/ac838d5e38f146fa9cc211ed9b73477e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/12ab330da38d4780bb6d4ab1e6ae2178, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/8af59a589ae44a3b89db942bd475d100, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/8d13eb3f76c0471d864f09d9ce4762c7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/140ce797b527429ba679afdf806f89aa, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/5188f9e66a2d4bd1a6650e387863a122, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/4025bc6b0eb64f26bdd00b4beb44c896, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/912472bd580a4a10b940865b8f9989ae, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/a88bf3e67cb541b7bf51498d79245c22, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/d4b177b2b4d041789b182d242624d1eb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/6134fad50e304bd790780482abd89dbc, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/44644f782ba14ff88bfe4f4c7342a162] to archive 2024-11-11T07:30:14,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:30:14,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/82d8c2e6825b414d93b6edd0c287638d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/82d8c2e6825b414d93b6edd0c287638d 2024-11-11T07:30:14,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/1ebd3ac1149a4e13887f440e2ae79e92 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/1ebd3ac1149a4e13887f440e2ae79e92 2024-11-11T07:30:14,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/f9d9421655cc446f920930dc2adb32cf to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/f9d9421655cc446f920930dc2adb32cf 2024-11-11T07:30:14,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/b52402183a184fce929716b91a3ed5e9 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/b52402183a184fce929716b91a3ed5e9 2024-11-11T07:30:14,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/85db1f5653cc46a78e32176521a5ae02 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/85db1f5653cc46a78e32176521a5ae02 2024-11-11T07:30:14,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/03d20c5fc85547698aaaf983cbee57ed to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/03d20c5fc85547698aaaf983cbee57ed 2024-11-11T07:30:14,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/6e417d57a4274f4daf0944e6b0ac35dd to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/6e417d57a4274f4daf0944e6b0ac35dd 2024-11-11T07:30:14,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/e5d95791a6334f38a96b2a163c6686a1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/e5d95791a6334f38a96b2a163c6686a1 2024-11-11T07:30:14,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/ac838d5e38f146fa9cc211ed9b73477e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/ac838d5e38f146fa9cc211ed9b73477e 2024-11-11T07:30:14,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/12ab330da38d4780bb6d4ab1e6ae2178 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/12ab330da38d4780bb6d4ab1e6ae2178 2024-11-11T07:30:14,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/8af59a589ae44a3b89db942bd475d100 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/8af59a589ae44a3b89db942bd475d100 2024-11-11T07:30:14,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/8d13eb3f76c0471d864f09d9ce4762c7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/8d13eb3f76c0471d864f09d9ce4762c7 2024-11-11T07:30:14,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/140ce797b527429ba679afdf806f89aa to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/140ce797b527429ba679afdf806f89aa 2024-11-11T07:30:14,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/5188f9e66a2d4bd1a6650e387863a122 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/5188f9e66a2d4bd1a6650e387863a122 2024-11-11T07:30:14,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/4025bc6b0eb64f26bdd00b4beb44c896 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/4025bc6b0eb64f26bdd00b4beb44c896 2024-11-11T07:30:14,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/912472bd580a4a10b940865b8f9989ae to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/912472bd580a4a10b940865b8f9989ae 2024-11-11T07:30:14,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/a88bf3e67cb541b7bf51498d79245c22 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/a88bf3e67cb541b7bf51498d79245c22 2024-11-11T07:30:14,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/d4b177b2b4d041789b182d242624d1eb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/d4b177b2b4d041789b182d242624d1eb 2024-11-11T07:30:14,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/6134fad50e304bd790780482abd89dbc to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/6134fad50e304bd790780482abd89dbc 2024-11-11T07:30:14,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/44644f782ba14ff88bfe4f4c7342a162 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/44644f782ba14ff88bfe4f4c7342a162 2024-11-11T07:30:14,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/4fcfcaef13f24c4098e67b2fa9988245, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/c476f1d305f54245a21e6fcb2ad2ae06, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/1c3dcc1b822349e0a3760c938e814e1b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/3d3cfd10d43d4010905550500b6f04db, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/347fc3c22816425c8be6363fd174f1a7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/873b065458e346439380207534fc51cf, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/2ea2e13b6f3d47aaa75b2bc7faba38e4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/732352b7793848c48ee5c82821ede75b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/cb947a4ef75741f5bd2f59576b2c4294, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/603fe0ed84f14777a82e5c11e362b957, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/28450c8d06b44c63b71c1ca18289f84a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/27869486f23d431da4dd204119d4f69a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/a732fe8d1b4040e293705d2bc233734a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/98dff9632b7043b7a4dd28bc9bb41d6f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/b909e28f74b34ea595d7855bd40f5c0f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/273fe1175a8044208d95c5e48aed78a6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/1749c85467b14b699cbc012a14aedf9f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/55775e4eaf9944458c3bc5d10e74fa16, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/762ac753f73d4cdea41388f78d9f42a0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/6e1e6061a0d54201a567974c5a1d36af] to archive 2024-11-11T07:30:14,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:30:14,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/4fcfcaef13f24c4098e67b2fa9988245 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/4fcfcaef13f24c4098e67b2fa9988245 2024-11-11T07:30:14,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/c476f1d305f54245a21e6fcb2ad2ae06 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/c476f1d305f54245a21e6fcb2ad2ae06 2024-11-11T07:30:14,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/1c3dcc1b822349e0a3760c938e814e1b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/1c3dcc1b822349e0a3760c938e814e1b 2024-11-11T07:30:14,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/3d3cfd10d43d4010905550500b6f04db to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/3d3cfd10d43d4010905550500b6f04db 2024-11-11T07:30:14,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/347fc3c22816425c8be6363fd174f1a7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/347fc3c22816425c8be6363fd174f1a7 2024-11-11T07:30:14,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/873b065458e346439380207534fc51cf to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/873b065458e346439380207534fc51cf 2024-11-11T07:30:14,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/2ea2e13b6f3d47aaa75b2bc7faba38e4 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/2ea2e13b6f3d47aaa75b2bc7faba38e4 2024-11-11T07:30:14,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/732352b7793848c48ee5c82821ede75b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/732352b7793848c48ee5c82821ede75b 2024-11-11T07:30:14,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/cb947a4ef75741f5bd2f59576b2c4294 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/cb947a4ef75741f5bd2f59576b2c4294 2024-11-11T07:30:14,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/603fe0ed84f14777a82e5c11e362b957 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/603fe0ed84f14777a82e5c11e362b957 2024-11-11T07:30:14,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/28450c8d06b44c63b71c1ca18289f84a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/28450c8d06b44c63b71c1ca18289f84a 2024-11-11T07:30:14,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/27869486f23d431da4dd204119d4f69a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/27869486f23d431da4dd204119d4f69a 2024-11-11T07:30:14,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/a732fe8d1b4040e293705d2bc233734a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/a732fe8d1b4040e293705d2bc233734a 2024-11-11T07:30:14,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/98dff9632b7043b7a4dd28bc9bb41d6f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/98dff9632b7043b7a4dd28bc9bb41d6f 2024-11-11T07:30:14,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/b909e28f74b34ea595d7855bd40f5c0f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/b909e28f74b34ea595d7855bd40f5c0f 2024-11-11T07:30:14,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/273fe1175a8044208d95c5e48aed78a6 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/273fe1175a8044208d95c5e48aed78a6 2024-11-11T07:30:14,550 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/1749c85467b14b699cbc012a14aedf9f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/1749c85467b14b699cbc012a14aedf9f 2024-11-11T07:30:14,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/55775e4eaf9944458c3bc5d10e74fa16 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/55775e4eaf9944458c3bc5d10e74fa16 2024-11-11T07:30:14,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/762ac753f73d4cdea41388f78d9f42a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/762ac753f73d4cdea41388f78d9f42a0 2024-11-11T07:30:14,553 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/6e1e6061a0d54201a567974c5a1d36af to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/6e1e6061a0d54201a567974c5a1d36af 2024-11-11T07:30:14,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/9104fce7b44b463ab9f1c730173748ee, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/692a372fa9d1456da7db589eb99874c4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/79898e5d0dfe44ea850abda2610d8379, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/1cdb0eead393426ca43ca26c402923e4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/0676dd95de714ee1820f90c6bf3b82c8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/d564f4a5b62b4280b3e893547cf4fa3c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/218a10e1b58147428ab247fa9f0c4642, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/f50c3e48aeb14365a814171f879751e9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/88b3852a82eb4ef3ad9696c3160b60d1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/5691605df52c469eb114daa05a752408, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/e5124c2a0ba9441caacabda265eb598d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/ef21f379dd334c23beb2300102a3342c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/c9d5a9853da04a39872b1ca68e930add, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/879e20cbe2e84fb4824810bed58e1ed5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/76d2172f2c684083aee0738fdd16a87a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/9fb78e3d7ea0473ba7b321a92839a3c9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/f58dad78a6ac4757810973d7cb2d1565, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/93ef7caa93c243e7845934f1f40ea181, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/d5f666f7010e464e90ac898ab0270899, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/87abbd0046d54d80b7218435a82dcd67] to archive 2024-11-11T07:30:14,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:30:14,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/9104fce7b44b463ab9f1c730173748ee to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/9104fce7b44b463ab9f1c730173748ee 2024-11-11T07:30:14,558 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/692a372fa9d1456da7db589eb99874c4 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/692a372fa9d1456da7db589eb99874c4 2024-11-11T07:30:14,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/79898e5d0dfe44ea850abda2610d8379 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/79898e5d0dfe44ea850abda2610d8379 2024-11-11T07:30:14,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/1cdb0eead393426ca43ca26c402923e4 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/1cdb0eead393426ca43ca26c402923e4 2024-11-11T07:30:14,565 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/0676dd95de714ee1820f90c6bf3b82c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/0676dd95de714ee1820f90c6bf3b82c8 2024-11-11T07:30:14,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/d564f4a5b62b4280b3e893547cf4fa3c to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/d564f4a5b62b4280b3e893547cf4fa3c 2024-11-11T07:30:14,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/218a10e1b58147428ab247fa9f0c4642 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/218a10e1b58147428ab247fa9f0c4642 2024-11-11T07:30:14,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/f50c3e48aeb14365a814171f879751e9 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/f50c3e48aeb14365a814171f879751e9 2024-11-11T07:30:14,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/88b3852a82eb4ef3ad9696c3160b60d1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/88b3852a82eb4ef3ad9696c3160b60d1 2024-11-11T07:30:14,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/5691605df52c469eb114daa05a752408 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/5691605df52c469eb114daa05a752408 2024-11-11T07:30:14,573 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/e5124c2a0ba9441caacabda265eb598d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/e5124c2a0ba9441caacabda265eb598d 2024-11-11T07:30:14,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/ef21f379dd334c23beb2300102a3342c to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/ef21f379dd334c23beb2300102a3342c 2024-11-11T07:30:14,576 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/c9d5a9853da04a39872b1ca68e930add to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/c9d5a9853da04a39872b1ca68e930add 2024-11-11T07:30:14,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/879e20cbe2e84fb4824810bed58e1ed5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/879e20cbe2e84fb4824810bed58e1ed5 2024-11-11T07:30:14,579 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/76d2172f2c684083aee0738fdd16a87a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/76d2172f2c684083aee0738fdd16a87a 2024-11-11T07:30:14,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/9fb78e3d7ea0473ba7b321a92839a3c9 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/9fb78e3d7ea0473ba7b321a92839a3c9 2024-11-11T07:30:14,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/f58dad78a6ac4757810973d7cb2d1565 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/f58dad78a6ac4757810973d7cb2d1565 2024-11-11T07:30:14,582 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/93ef7caa93c243e7845934f1f40ea181 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/93ef7caa93c243e7845934f1f40ea181 2024-11-11T07:30:14,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/d5f666f7010e464e90ac898ab0270899 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/d5f666f7010e464e90ac898ab0270899 2024-11-11T07:30:14,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/87abbd0046d54d80b7218435a82dcd67 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/87abbd0046d54d80b7218435a82dcd67 2024-11-11T07:30:14,589 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/recovered.edits/323.seqid, newMaxSeqId=323, maxSeqId=4 2024-11-11T07:30:14,590 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. 2024-11-11T07:30:14,590 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] regionserver.HRegion(1676): Region close journal for 18312f0c839a758e8897bb39c81d93ff: Waiting for close lock at 1731310212830Running coprocessor pre-close hooks at 1731310212830Disabling compacts and flushes for region at 1731310212830Disabling writes for close at 1731310212830Obtaining lock to block concurrent updates at 1731310212831 (+1 ms)Preparing flush snapshotting stores in 18312f0c839a758e8897bb39c81d93ff at 1731310212831Finished memstore snapshotting TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731310212831Flushing stores of TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff. at 1731310212832 (+1 ms)Flushing 18312f0c839a758e8897bb39c81d93ff/A: creating writer at 1731310212832Flushing mob file 18312f0c839a758e8897bb39c81d93ff/A: appending metadata at 1731310212837 (+5 ms)Flushing mob file 18312f0c839a758e8897bb39c81d93ff/A: closing flushed file at 1731310212837Flushing 18312f0c839a758e8897bb39c81d93ff/A: appending metadata at 1731310213248 (+411 ms)Flushing 18312f0c839a758e8897bb39c81d93ff/A: closing flushed file at 1731310213248Flushing 18312f0c839a758e8897bb39c81d93ff/B: creating writer at 1731310213660 (+412 ms)Flushing 18312f0c839a758e8897bb39c81d93ff/B: appending metadata at 1731310213663 (+3 ms)Flushing 18312f0c839a758e8897bb39c81d93ff/B: closing flushed file at 1731310213663Flushing 18312f0c839a758e8897bb39c81d93ff/C: creating writer at 1731310214072 (+409 ms)Flushing 18312f0c839a758e8897bb39c81d93ff/C: appending metadata at 1731310214075 (+3 ms)Flushing 18312f0c839a758e8897bb39c81d93ff/C: closing flushed file at 1731310214075Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b075bd: reopening flushed file at 1731310214485 (+410 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@224829a2: reopening flushed file at 1731310214490 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@113a769: reopening flushed file at 1731310214496 (+6 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 18312f0c839a758e8897bb39c81d93ff in 1671ms, sequenceid=320, compaction requested=true at 1731310214502 (+6 ms)Writing region close event to WAL at 1731310214586 (+84 ms)Running coprocessor post-close hooks at 1731310214590 (+4 ms)Closed at 1731310214590 2024-11-11T07:30:14,593 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=55}] handler.UnassignRegionHandler(157): Closed 18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,593 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=54 updating hbase:meta row=18312f0c839a758e8897bb39c81d93ff, regionState=CLOSED 2024-11-11T07:30:14,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=55, ppid=54, state=RUNNABLE, hasLock=false; CloseRegionProcedure 18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:30:14,598 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-11T07:30:14,598 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; CloseRegionProcedure 18312f0c839a758e8897bb39c81d93ff, server=ce2cf0244921,33345,1731310155369 in 1.9190 sec 2024-11-11T07:30:14,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=54, resume processing ppid=53 2024-11-11T07:30:14,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, ppid=53, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=18312f0c839a758e8897bb39c81d93ff, UNASSIGN in 1.9240 sec 2024-11-11T07:30:14,604 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310214604"}]},"ts":"1731310214604"} 2024-11-11T07:30:14,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-11T07:30:14,605 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9280 sec 2024-11-11T07:30:14,606 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T07:30:14,606 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T07:30:14,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9400 sec 2024-11-11T07:30:14,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-11T07:30:14,815 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:14,816 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T07:30:14,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:14,817 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=56, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:14,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T07:30:14,819 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=56, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:14,821 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,824 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/recovered.edits] 2024-11-11T07:30:14,826 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/02eb9ba45df0467cb7341904616c17b5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/02eb9ba45df0467cb7341904616c17b5 2024-11-11T07:30:14,827 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/b8d80ca6d3be4e0cbf37f1153745d44a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/b8d80ca6d3be4e0cbf37f1153745d44a 2024-11-11T07:30:14,829 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/cd8e52593b0f4b5bb7e613007d98fc88 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/cd8e52593b0f4b5bb7e613007d98fc88 2024-11-11T07:30:14,830 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/e248aa534ee749b48c90356dfd5314f6 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/A/e248aa534ee749b48c90356dfd5314f6 2024-11-11T07:30:14,832 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/3776e761da664d89b1aa94a56ffff728 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/3776e761da664d89b1aa94a56ffff728 2024-11-11T07:30:14,833 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/a165b871a7b042eab0da8dd8fee329fe to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/a165b871a7b042eab0da8dd8fee329fe 2024-11-11T07:30:14,835 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/c97d85c8c80344279c8ac71e6f6221d6 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/c97d85c8c80344279c8ac71e6f6221d6 2024-11-11T07:30:14,836 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/e1d9142021794fa18c558d3318960962 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/B/e1d9142021794fa18c558d3318960962 2024-11-11T07:30:14,839 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/30224cabf8874257a43c185c697cb2fe to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/30224cabf8874257a43c185c697cb2fe 2024-11-11T07:30:14,840 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/5d5220e2b3504c8d9319feb3481e43ec to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/5d5220e2b3504c8d9319feb3481e43ec 2024-11-11T07:30:14,842 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/b02d61ec4d6941c9ba9aec233cc62896 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/b02d61ec4d6941c9ba9aec233cc62896 2024-11-11T07:30:14,843 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/d7a2c92c17e84bf585b26d6c39c3cdb3 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/C/d7a2c92c17e84bf585b26d6c39c3cdb3 2024-11-11T07:30:14,846 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/recovered.edits/323.seqid to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff/recovered.edits/323.seqid 2024-11-11T07:30:14,847 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,847 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T07:30:14,847 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T07:30:14,848 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T07:30:14,853 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111cc3be004d8b4084a261bf58920a9686_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111cc3be004d8b4084a261bf58920a9686_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,854 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111ebc296e6213416abc0e510e1c4e0709_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411111ebc296e6213416abc0e510e1c4e0709_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,855 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112a6fd4e208c3486abb528f36d2bf61c7_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411112a6fd4e208c3486abb528f36d2bf61c7_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,857 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113786f4291ed24041a135a8a49905c977_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411113786f4291ed24041a135a8a49905c977_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,858 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111145e612c2cbbc429b999145e93c546001_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111145e612c2cbbc429b999145e93c546001_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,859 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111497e3ef2e2a84369ac121c8cf7b89ec7_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111497e3ef2e2a84369ac121c8cf7b89ec7_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,861 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114d403f5de52e4f5e812dce1d08155f9e_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114d403f5de52e4f5e812dce1d08155f9e_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,862 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116308b506564b4381b1e9f7a64214eff8_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116308b506564b4381b1e9f7a64214eff8_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,863 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116b7412820f07445599c3ab0867132684_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411116b7412820f07445599c3ab0867132684_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,865 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111170953c139cd14a7a8e0fd61b8bc8c47d_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111170953c139cd14a7a8e0fd61b8bc8c47d_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,866 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117df7957dc3be4c568cd8d0699cb6c753_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411117df7957dc3be4c568cd8d0699cb6c753_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,867 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111182c307212c2c4d82a71bf2ecb2f0e81a_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111182c307212c2c4d82a71bf2ecb2f0e81a_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,868 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118e03be549a044c29b2f60ec173e0499f_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118e03be549a044c29b2f60ec173e0499f_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,870 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a29bd16327d8417a877fe42d89e13dc3_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a29bd16327d8417a877fe42d89e13dc3_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,871 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111aa21bf43ef0b4330a748429ba9638d45_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111aa21bf43ef0b4330a748429ba9638d45_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,872 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ccb596bc50044dd4a38a150b42e1924d_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ccb596bc50044dd4a38a150b42e1924d_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,873 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ece2b2dc25654321bb743af5e2c9eb80_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ece2b2dc25654321bb743af5e2c9eb80_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,874 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f53dca361d8e4f5287173c3a50f3adb9_18312f0c839a758e8897bb39c81d93ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f53dca361d8e4f5287173c3a50f3adb9_18312f0c839a758e8897bb39c81d93ff 2024-11-11T07:30:14,875 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T07:30:14,877 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=56, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:14,880 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T07:30:14,882 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T07:30:14,883 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=56, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:14,884 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T07:30:14,884 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731310214884"}]},"ts":"9223372036854775807"} 2024-11-11T07:30:14,886 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T07:30:14,886 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 18312f0c839a758e8897bb39c81d93ff, NAME => 'TestAcidGuarantees,,1731310187361.18312f0c839a758e8897bb39c81d93ff.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T07:30:14,886 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T07:30:14,887 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731310214886"}]},"ts":"9223372036854775807"} 2024-11-11T07:30:14,889 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T07:30:14,889 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=56, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:14,891 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T07:30:14,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 73 msec 2024-11-11T07:30:14,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-11T07:30:14,925 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T07:30:14,925 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:14,936 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=251 (was 248) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_503664647_22 at /127.0.0.1:60340 [Waiting for operation #70] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_606034174_22 at /127.0.0.1:44894 [Waiting for operation #203] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_606034174_22 at /127.0.0.1:44774 [Waiting for operation #224] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/cluster_43578dcd-0ad0-2e50-c990-4483826cf543/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/cluster_43578dcd-0ad0-2e50-c990-4483826cf543/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=926 (was 838) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2581 (was 3257) 2024-11-11T07:30:14,946 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=251, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=926, ProcessCount=11, AvailableMemoryMB=2581 2024-11-11T07:30:14,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:30:14,948 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:30:14,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:14,950 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T07:30:14,950 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:14,950 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 57 2024-11-11T07:30:14,951 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T07:30:14,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T07:30:14,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742050_1226 (size=1061) 2024-11-11T07:30:15,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T07:30:15,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T07:30:15,381 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e 2024-11-11T07:30:15,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742051_1227 (size=53) 2024-11-11T07:30:15,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T07:30:15,787 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:30:15,787 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 385162ecea32f0e101e7709026aea910, disabling compactions & flushes 2024-11-11T07:30:15,787 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:15,787 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:15,787 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. after waiting 0 ms 2024-11-11T07:30:15,788 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:15,788 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:15,788 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 385162ecea32f0e101e7709026aea910: Waiting for close lock at 1731310215787Disabling compacts and flushes for region at 1731310215787Disabling writes for close at 1731310215788 (+1 ms)Writing region close event to WAL at 1731310215788Closed at 1731310215788 2024-11-11T07:30:15,789 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T07:30:15,789 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731310215789"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731310215789"}]},"ts":"1731310215789"} 2024-11-11T07:30:15,791 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T07:30:15,791 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T07:30:15,792 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310215791"}]},"ts":"1731310215791"} 2024-11-11T07:30:15,793 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T07:30:15,793 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=385162ecea32f0e101e7709026aea910, ASSIGN}] 2024-11-11T07:30:15,795 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=385162ecea32f0e101e7709026aea910, ASSIGN 2024-11-11T07:30:15,796 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=385162ecea32f0e101e7709026aea910, ASSIGN; state=OFFLINE, location=ce2cf0244921,33345,1731310155369; forceNewPlan=false, retain=false 2024-11-11T07:30:15,946 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=385162ecea32f0e101e7709026aea910, regionState=OPENING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:15,948 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=58, ppid=57, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=385162ecea32f0e101e7709026aea910, ASSIGN because future has completed 2024-11-11T07:30:15,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:30:16,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T07:30:16,104 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:16,104 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7752): Opening region: {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:30:16,104 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:16,104 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:30:16,104 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7794): checking encryption for 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:16,104 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(7797): checking classloading for 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:16,109 INFO [StoreOpener-385162ecea32f0e101e7709026aea910-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:16,110 INFO [StoreOpener-385162ecea32f0e101e7709026aea910-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:30:16,110 INFO [StoreOpener-385162ecea32f0e101e7709026aea910-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 385162ecea32f0e101e7709026aea910 columnFamilyName A 2024-11-11T07:30:16,110 DEBUG [StoreOpener-385162ecea32f0e101e7709026aea910-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:16,111 INFO [StoreOpener-385162ecea32f0e101e7709026aea910-1 {}] regionserver.HStore(327): Store=385162ecea32f0e101e7709026aea910/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:30:16,111 INFO [StoreOpener-385162ecea32f0e101e7709026aea910-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:16,112 INFO [StoreOpener-385162ecea32f0e101e7709026aea910-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:30:16,112 INFO [StoreOpener-385162ecea32f0e101e7709026aea910-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 385162ecea32f0e101e7709026aea910 columnFamilyName B 2024-11-11T07:30:16,112 DEBUG [StoreOpener-385162ecea32f0e101e7709026aea910-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:16,113 INFO [StoreOpener-385162ecea32f0e101e7709026aea910-1 {}] regionserver.HStore(327): Store=385162ecea32f0e101e7709026aea910/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:30:16,113 INFO [StoreOpener-385162ecea32f0e101e7709026aea910-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:16,114 INFO [StoreOpener-385162ecea32f0e101e7709026aea910-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:30:16,114 INFO [StoreOpener-385162ecea32f0e101e7709026aea910-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 385162ecea32f0e101e7709026aea910 columnFamilyName C 2024-11-11T07:30:16,114 DEBUG [StoreOpener-385162ecea32f0e101e7709026aea910-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:16,114 INFO [StoreOpener-385162ecea32f0e101e7709026aea910-1 {}] regionserver.HStore(327): Store=385162ecea32f0e101e7709026aea910/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:30:16,114 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:16,114 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1038): replaying wal for 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:16,115 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910 2024-11-11T07:30:16,115 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910 2024-11-11T07:30:16,116 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1048): stopping wal replay for 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:16,116 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1060): Cleaning up temporary data for 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:16,117 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:30:16,118 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1093): writing seq id for 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:16,119 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T07:30:16,120 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1114): Opened 385162ecea32f0e101e7709026aea910; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69888110, jitterRate=0.04141399264335632}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:30:16,120 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:16,121 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegion(1006): Region open journal for 385162ecea32f0e101e7709026aea910: Running coprocessor pre-open hook at 1731310216104Writing region info on filesystem at 1731310216104Initializing all the Stores at 1731310216105 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310216105Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310216108 (+3 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310216108Cleaning up temporary data from old regions at 1731310216116 (+8 ms)Running coprocessor post-open hooks at 1731310216120 (+4 ms)Region opened successfully at 1731310216121 (+1 ms) 2024-11-11T07:30:16,122 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., pid=59, masterSystemTime=1731310216100 2024-11-11T07:30:16,123 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:16,123 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=59}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:16,124 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=58 updating hbase:meta row=385162ecea32f0e101e7709026aea910, regionState=OPEN, openSeqNum=2, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:16,125 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=59, ppid=58, state=RUNNABLE, hasLock=false; OpenRegionProcedure 385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:30:16,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-11T07:30:16,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; OpenRegionProcedure 385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 in 178 msec 2024-11-11T07:30:16,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-11-11T07:30:16,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=385162ecea32f0e101e7709026aea910, ASSIGN in 335 msec 2024-11-11T07:30:16,130 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T07:30:16,130 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310216130"}]},"ts":"1731310216130"} 2024-11-11T07:30:16,132 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T07:30:16,132 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=57, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T07:30:16,134 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1850 sec 2024-11-11T07:30:17,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-11-11T07:30:17,095 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:17,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fe11fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,096 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:17,096 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:17,097 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:17,097 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:17,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@10942c43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:17,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:17,098 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:17,099 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57906, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:17,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5097cda6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,100 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:17,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:17,102 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49910, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:17,103 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T07:30:17,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is ce2cf0244921,33029,1731310154427 2024-11-11T07:30:17,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1c15b409 2024-11-11T07:30:17,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T07:30:17,105 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T07:30:17,107 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34b5b273, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,107 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:17,107 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:17,108 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@104550, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:17,108 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:17,109 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:17,109 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:17,110 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a7d892c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1240be3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,111 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:17,111 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:17,111 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d57cdcb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:17,112 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:17,113 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57950, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:17,113 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@380e6387, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,114 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2885eb44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,114 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:17,114 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:17,115 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:17,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:17,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:17,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63d08a52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,115 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:17,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:17,116 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:17,117 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:17,117 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b4df8cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,118 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692a6227, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,118 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:17,118 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:17,119 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:17,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:17,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:17,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@209a74ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,119 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:17,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:17,120 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:17,121 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:17,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37e33283, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,124 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@128d97bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,124 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:17,124 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:17,125 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:17,125 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:17,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ebf399e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:17,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:17,126 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:17,127 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:17,127 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33e8f576, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626de388, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,128 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:17,128 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:17,129 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@462050e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:17,133 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:17,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5be552, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,135 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47a5c877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,135 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:17,135 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:17,136 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18327c39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:17,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:17,138 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58048, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@231aea88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@679f7426, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,139 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:17,139 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:17,140 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6147b70c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:17,140 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:17,141 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:17,142 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ff8914f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,142 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b71af22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,142 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:17,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:17,143 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:17,143 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f900624, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:17,145 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58094, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:17,145 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@607d88b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,146 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@728d338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,146 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:17,146 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:17,147 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@331f7bc8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:17,147 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:17,148 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:17,149 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cdc95ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:17,149 DEBUG [Thread-1070 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:17,149 DEBUG [Thread-1079 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:17,150 DEBUG [Thread-1077 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:17,150 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:17,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:17,150 DEBUG [Thread-1083 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:17,151 DEBUG [Thread-1066 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:17,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:17,153 DEBUG [Thread-1085 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:17,153 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49920, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:17,154 DEBUG [Thread-1068 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:17,155 DEBUG [Thread-1074 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:17,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:30:17,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:17,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49942, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:17,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:17,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49928, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:17,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49964, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:17,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:17,162 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:17,163 DEBUG [Thread-1072 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:17,164 DEBUG [Thread-1072 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:17,164 DEBUG [Thread-1072 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:17,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:17,166 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:30:17,166 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:30:17,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49974, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:30:17,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:30:17,169 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:30:17,169 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:30:17,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:30:17,175 DEBUG [Thread-1081 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:17,176 DEBUG [Thread-1081 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:17,176 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:30:17,176 DEBUG [Thread-1081 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:17,177 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49986, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:17,179 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:30:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-11T07:30:17,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T07:30:17,194 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:17,195 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:17,195 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:17,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T07:30:17,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:17,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:17,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:17,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:17,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:17,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:17,207 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:17,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=178, cellSize=8148, totalHeapSize=18752, min timestamp=1731310217276, max timestamp=1731310217291Num uniques 859; 2024-11-11T07:30:17,299 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39334, new segment data size=8148, suffix heap size=117064,new segment heap size=18752 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=178 2024-11-11T07:30:17,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310217276, max timestamp=1731310217292Num uniques 859; 2024-11-11T07:30:17,301 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:30:17,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T07:30:17,305 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/edabf3d46e894cc792cfbc3e65580ec0 is 50, key is test_row_0/A:col10/1731310217199/Put/seqid=0 2024-11-11T07:30:17,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310217276, max timestamp=1731310217292Num uniques 859; 2024-11-11T07:30:17,311 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:30:17,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742052_1228 (size=12001) 2024-11-11T07:30:17,355 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T07:30:17,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:17,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:17,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:17,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:17,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:17,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:17,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:17,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310277347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:17,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310277352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:17,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:17,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310277356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:17,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310277361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:17,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:17,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310277362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,373 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:17,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:17,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310277464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:17,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:17,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310277474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:17,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:17,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310277474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,481 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:17,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:17,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310277485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:17,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:17,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310277485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,491 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:17,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T07:30:17,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:17,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:17,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:17,511 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:17,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:17,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:17,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T07:30:17,672 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T07:30:17,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:17,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:17,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:17,673 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:17,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:17,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:17,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:17,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310277675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:17,686 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310277684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:17,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:17,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310277687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,693 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:17,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:17,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310277696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:17,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:17,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:17,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310277696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,711 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:17,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/edabf3d46e894cc792cfbc3e65580ec0 2024-11-11T07:30:17,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/f8a602eb2d15411d9caf4fa69732509a is 50, key is test_row_0/B:col10/1731310217199/Put/seqid=0 2024-11-11T07:30:17,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742053_1229 (size=12001) 2024-11-11T07:30:17,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T07:30:17,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T07:30:17,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:17,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:17,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:17,826 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:17,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:17,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:17,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T07:30:17,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:17,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:17,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:17,995 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:17,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:17,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310277985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:17,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:17,995 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:17,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:18,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310277995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,001 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,011 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310278005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,012 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,012 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310278007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,015 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310278019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,023 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T07:30:18,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:18,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:18,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:18,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:18,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:18,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:18,225 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/f8a602eb2d15411d9caf4fa69732509a 2024-11-11T07:30:18,286 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/adf43b21beb24c3a8d86a505fc4747a0 is 50, key is test_row_0/C:col10/1731310217199/Put/seqid=0 2024-11-11T07:30:18,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T07:30:18,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:18,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:18,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:18,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:18,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:18,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742054_1230 (size=12001) 2024-11-11T07:30:18,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:18,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/adf43b21beb24c3a8d86a505fc4747a0 2024-11-11T07:30:18,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T07:30:18,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/edabf3d46e894cc792cfbc3e65580ec0 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/edabf3d46e894cc792cfbc3e65580ec0 2024-11-11T07:30:18,394 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/edabf3d46e894cc792cfbc3e65580ec0, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T07:30:18,407 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/f8a602eb2d15411d9caf4fa69732509a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f8a602eb2d15411d9caf4fa69732509a 2024-11-11T07:30:18,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f8a602eb2d15411d9caf4fa69732509a, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T07:30:18,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/adf43b21beb24c3a8d86a505fc4747a0 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/adf43b21beb24c3a8d86a505fc4747a0 2024-11-11T07:30:18,500 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/adf43b21beb24c3a8d86a505fc4747a0, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T07:30:18,511 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=128.32 KB/131404 for 385162ecea32f0e101e7709026aea910 in 1312ms, sequenceid=12, compaction requested=false 2024-11-11T07:30:18,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-11T07:30:18,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:18,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:18,511 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=128.32 KB heapSize=366.99 KB 2024-11-11T07:30:18,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:18,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:18,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:18,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:18,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:18,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:18,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:18,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:18,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/5fb6095ddd98453f8f3dc2e06af3b778 is 50, key is test_row_0/A:col10/1731310217356/Put/seqid=0 2024-11-11T07:30:18,559 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310278543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310278546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310278551, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,560 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310278553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,563 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,563 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,563 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,567 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310278567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,583 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742055_1231 (size=12001) 2024-11-11T07:30:18,669 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310278665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,670 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310278667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310278675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310278679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310278694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310278884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,888 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310278884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,889 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,895 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310278886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,899 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310278895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:18,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:18,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310278915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:18,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:18,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:19,018 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=43.34 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/5fb6095ddd98453f8f3dc2e06af3b778 2024-11-11T07:30:19,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/f57b0cdc346b4b6b80abc1ca81732b43 is 50, key is test_row_0/B:col10/1731310217356/Put/seqid=0 2024-11-11T07:30:19,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742056_1232 (size=12001) 2024-11-11T07:30:19,123 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/f57b0cdc346b4b6b80abc1ca81732b43 2024-11-11T07:30:19,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/e9e0d1efb0cb4c5695cccffdc2c84eb8 is 50, key is test_row_0/C:col10/1731310217356/Put/seqid=0 2024-11-11T07:30:19,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:19,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310279195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,203 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:19,208 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:19,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310279199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:19,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:19,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310279205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:19,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:19,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:19,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310279215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:19,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742057_1233 (size=12001) 2024-11-11T07:30:19,244 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310279227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:19,246 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/e9e0d1efb0cb4c5695cccffdc2c84eb8 2024-11-11T07:30:19,250 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:19,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/5fb6095ddd98453f8f3dc2e06af3b778 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/5fb6095ddd98453f8f3dc2e06af3b778 2024-11-11T07:30:19,335 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/5fb6095ddd98453f8f3dc2e06af3b778, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T07:30:19,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/f57b0cdc346b4b6b80abc1ca81732b43 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f57b0cdc346b4b6b80abc1ca81732b43 2024-11-11T07:30:19,343 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f57b0cdc346b4b6b80abc1ca81732b43, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T07:30:19,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/e9e0d1efb0cb4c5695cccffdc2c84eb8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e9e0d1efb0cb4c5695cccffdc2c84eb8 2024-11-11T07:30:19,351 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e9e0d1efb0cb4c5695cccffdc2c84eb8, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T07:30:19,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T07:30:19,355 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~128.32 KB/131404, heapSize ~366.95 KB/375752, currentSize=53.67 KB/54960 for 385162ecea32f0e101e7709026aea910 in 844ms, sequenceid=48, compaction requested=false 2024-11-11T07:30:19,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:19,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:19,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-11T07:30:19,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-11T07:30:19,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-11T07:30:19,361 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1630 sec 2024-11-11T07:30:19,365 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.2130 sec 2024-11-11T07:30:19,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:19,720 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T07:30:19,727 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:19,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:19,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:19,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:19,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:19,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:19,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/3930a3c12d4643ccbc44da7869934d46 is 50, key is test_row_0/A:col10/1731310219717/Put/seqid=0 2024-11-11T07:30:19,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742058_1234 (size=14341) 2024-11-11T07:30:19,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:19,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310279771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:19,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:19,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310279783, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:19,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310279784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,792 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:19,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:19,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310279784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:19,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,793 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:19,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:19,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310279787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:19,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,794 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,795 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:19,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310279895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:19,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:19,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:19,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310279895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:19,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:19,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310279895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:19,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,900 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:19,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310279905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:19,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:19,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:19,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310279906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:19,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,914 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:19,915 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:20,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310280105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:20,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:20,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:20,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310280106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:20,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310280105, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:20,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:20,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310280114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,120 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:20,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:20,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310280127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,135 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:20,154 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T07:30:20,178 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/3930a3c12d4643ccbc44da7869934d46 2024-11-11T07:30:20,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/25a46c73df9c445f8ee8a72eb1e3469a is 50, key is test_row_0/B:col10/1731310219717/Put/seqid=0 2024-11-11T07:30:20,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742059_1235 (size=12001) 2024-11-11T07:30:20,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/25a46c73df9c445f8ee8a72eb1e3469a 2024-11-11T07:30:20,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/2a4cc419861a499281d56f02f2bd1329 is 50, key is test_row_0/C:col10/1731310219717/Put/seqid=0 2024-11-11T07:30:20,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742060_1236 (size=12001) 2024-11-11T07:30:20,421 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:20,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310280418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:20,422 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:20,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:20,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310280427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:20,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:20,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310280427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:20,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310280427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,434 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:20,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:20,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310280444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:20,446 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:20,799 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/2a4cc419861a499281d56f02f2bd1329 2024-11-11T07:30:20,830 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/3930a3c12d4643ccbc44da7869934d46 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/3930a3c12d4643ccbc44da7869934d46 2024-11-11T07:30:20,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/3930a3c12d4643ccbc44da7869934d46, entries=200, sequenceid=61, filesize=14.0 K 2024-11-11T07:30:20,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/25a46c73df9c445f8ee8a72eb1e3469a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/25a46c73df9c445f8ee8a72eb1e3469a 2024-11-11T07:30:20,856 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/25a46c73df9c445f8ee8a72eb1e3469a, entries=150, sequenceid=61, filesize=11.7 K 2024-11-11T07:30:20,857 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/2a4cc419861a499281d56f02f2bd1329 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/2a4cc419861a499281d56f02f2bd1329 2024-11-11T07:30:20,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/2a4cc419861a499281d56f02f2bd1329, entries=150, sequenceid=61, filesize=11.7 K 2024-11-11T07:30:20,863 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 385162ecea32f0e101e7709026aea910 in 1143ms, sequenceid=61, compaction requested=true 2024-11-11T07:30:20,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:20,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:20,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:20,863 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:20,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:20,863 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:20,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:20,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:20,863 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:20,864 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:20,864 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/A is initiating minor compaction (all files) 2024-11-11T07:30:20,864 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/A in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:20,865 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:20,865 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/B is initiating minor compaction (all files) 2024-11-11T07:30:20,865 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/B in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:20,865 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f8a602eb2d15411d9caf4fa69732509a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f57b0cdc346b4b6b80abc1ca81732b43, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/25a46c73df9c445f8ee8a72eb1e3469a] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=35.2 K 2024-11-11T07:30:20,865 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/edabf3d46e894cc792cfbc3e65580ec0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/5fb6095ddd98453f8f3dc2e06af3b778, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/3930a3c12d4643ccbc44da7869934d46] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=37.4 K 2024-11-11T07:30:20,865 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f8a602eb2d15411d9caf4fa69732509a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731310217168 2024-11-11T07:30:20,866 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting edabf3d46e894cc792cfbc3e65580ec0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731310217168 2024-11-11T07:30:20,866 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f57b0cdc346b4b6b80abc1ca81732b43, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731310217350 2024-11-11T07:30:20,866 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5fb6095ddd98453f8f3dc2e06af3b778, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731310217350 2024-11-11T07:30:20,866 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 25a46c73df9c445f8ee8a72eb1e3469a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731310218535 2024-11-11T07:30:20,867 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3930a3c12d4643ccbc44da7869934d46, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731310218535 2024-11-11T07:30:20,886 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#B#compaction#195 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:20,886 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/7845ea38663546bfa442860a0a784dfd is 50, key is test_row_0/B:col10/1731310219717/Put/seqid=0 2024-11-11T07:30:20,902 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#A#compaction#196 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:20,903 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/64614399dfee4662965bc74dce12c5ff is 50, key is test_row_0/A:col10/1731310219717/Put/seqid=0 2024-11-11T07:30:20,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310219773, max timestamp=1731310220926Num uniques 859; 2024-11-11T07:30:20,930 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:30:20,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310219773, max timestamp=1731310220926Num uniques 859; 2024-11-11T07:30:20,935 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:30:20,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310219773, max timestamp=1731310220926Num uniques 859; 2024-11-11T07:30:20,938 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:30:20,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742061_1237 (size=12104) 2024-11-11T07:30:20,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742062_1238 (size=12104) 2024-11-11T07:30:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:20,951 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=53.67 KB heapSize=145.92 KB 2024-11-11T07:30:20,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:20,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:20,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:20,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:20,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:20,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:20,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/a195c28df8494a568744efc561dc1d96 is 50, key is test_row_0/A:col10/1731310220950/Put/seqid=0 2024-11-11T07:30:20,964 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/64614399dfee4662965bc74dce12c5ff as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/64614399dfee4662965bc74dce12c5ff 2024-11-11T07:30:20,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742063_1239 (size=16681) 2024-11-11T07:30:20,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/a195c28df8494a568744efc561dc1d96 2024-11-11T07:30:20,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/7ed42671330e49c6bc8562780c6c563c is 50, key is test_row_0/B:col10/1731310220950/Put/seqid=0 2024-11-11T07:30:20,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742064_1240 (size=12001) 2024-11-11T07:30:20,991 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/A of 385162ecea32f0e101e7709026aea910 into 64614399dfee4662965bc74dce12c5ff(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:20,991 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:20,991 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/A, priority=13, startTime=1731310220863; duration=0sec 2024-11-11T07:30:20,991 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:20,991 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:A 2024-11-11T07:30:20,992 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:20,994 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:20,994 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/C is initiating minor compaction (all files) 2024-11-11T07:30:20,994 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/C in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:20,995 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/adf43b21beb24c3a8d86a505fc4747a0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e9e0d1efb0cb4c5695cccffdc2c84eb8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/2a4cc419861a499281d56f02f2bd1329] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=35.2 K 2024-11-11T07:30:20,999 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting adf43b21beb24c3a8d86a505fc4747a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731310217168 2024-11-11T07:30:21,000 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9e0d1efb0cb4c5695cccffdc2c84eb8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731310217350 2024-11-11T07:30:21,003 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a4cc419861a499281d56f02f2bd1329, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731310218535 2024-11-11T07:30:21,024 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#C#compaction#199 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:21,024 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/984395307849466188682aa9834674dd is 50, key is test_row_0/C:col10/1731310219717/Put/seqid=0 2024-11-11T07:30:21,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731310221025, max timestamp=1731310221040Num uniques 859; 2024-11-11T07:30:21,044 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T07:30:21,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731310221025, max timestamp=1731310221042Num uniques 859; 2024-11-11T07:30:21,045 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T07:30:21,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731310221025, max timestamp=1731310221042Num uniques 859; 2024-11-11T07:30:21,047 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T07:30:21,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742065_1241 (size=12104) 2024-11-11T07:30:21,093 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/984395307849466188682aa9834674dd as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/984395307849466188682aa9834674dd 2024-11-11T07:30:21,100 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/C of 385162ecea32f0e101e7709026aea910 into 984395307849466188682aa9834674dd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:21,100 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:21,100 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/C, priority=13, startTime=1731310220863; duration=0sec 2024-11-11T07:30:21,100 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:21,100 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:C 2024-11-11T07:30:21,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310281086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,108 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,109 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310281092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310281099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,113 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,113 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310281099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,115 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310281112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,122 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,123 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310281214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,218 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310281214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,219 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,220 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310281224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,230 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310281225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310281225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,342 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/7845ea38663546bfa442860a0a784dfd as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/7845ea38663546bfa442860a0a784dfd 2024-11-11T07:30:21,360 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/B of 385162ecea32f0e101e7709026aea910 into 7845ea38663546bfa442860a0a784dfd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:21,360 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:21,360 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/B, priority=13, startTime=1731310220863; duration=0sec 2024-11-11T07:30:21,360 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:21,360 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:B 2024-11-11T07:30:21,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-11T07:30:21,374 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:21,375 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:21,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-11T07:30:21,378 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:21,379 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:21,379 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:21,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T07:30:21,386 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/7ed42671330e49c6bc8562780c6c563c 2024-11-11T07:30:21,405 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/c0d053258c42402d9694b721dc7c5bf9 is 50, key is test_row_0/C:col10/1731310220950/Put/seqid=0 2024-11-11T07:30:21,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310281425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,428 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310281425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,428 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,429 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310281435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310281435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310281435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,442 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742066_1242 (size=12001) 2024-11-11T07:30:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T07:30:21,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T07:30:21,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:21,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:21,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:21,535 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:21,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:21,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:21,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T07:30:21,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:21,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:21,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:21,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:21,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:21,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:21,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T07:30:21,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310281734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310281734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310281755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310281755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:21,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310281755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:21,763 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:21,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T07:30:21,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:21,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:21,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:21,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:21,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:21,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:21,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/c0d053258c42402d9694b721dc7c5bf9 2024-11-11T07:30:21,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/a195c28df8494a568744efc561dc1d96 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/a195c28df8494a568744efc561dc1d96 2024-11-11T07:30:21,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/a195c28df8494a568744efc561dc1d96, entries=250, sequenceid=86, filesize=16.3 K 2024-11-11T07:30:21,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/7ed42671330e49c6bc8562780c6c563c as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/7ed42671330e49c6bc8562780c6c563c 2024-11-11T07:30:21,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/7ed42671330e49c6bc8562780c6c563c, entries=150, sequenceid=86, filesize=11.7 K 2024-11-11T07:30:21,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/c0d053258c42402d9694b721dc7c5bf9 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/c0d053258c42402d9694b721dc7c5bf9 2024-11-11T07:30:21,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/c0d053258c42402d9694b721dc7c5bf9, entries=150, sequenceid=86, filesize=11.7 K 2024-11-11T07:30:21,889 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~145.88 KB/149376, currentSize=134.18 KB/137400 for 385162ecea32f0e101e7709026aea910 in 938ms, sequenceid=86, compaction requested=false 2024-11-11T07:30:21,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:21,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-11T07:30:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:21,997 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-11T07:30:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:21,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:21,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:21,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:22,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T07:30:22,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/6e626442fbf04bb7b2f0533c94f2e672 is 50, key is test_row_0/A:col10/1731310221105/Put/seqid=0 2024-11-11T07:30:22,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742067_1243 (size=12001) 2024-11-11T07:30:22,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:22,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:22,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310282278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,286 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,287 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310282284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310282284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310282286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,293 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310282287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,293 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,294 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310282396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,402 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310282396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,402 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,409 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310282405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,410 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310282405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,410 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,410 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,457 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/6e626442fbf04bb7b2f0533c94f2e672 2024-11-11T07:30:22,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/5fe4c092e3214d2e98fbdac2a6db4ba1 is 50, key is test_row_0/B:col10/1731310221105/Put/seqid=0 2024-11-11T07:30:22,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T07:30:22,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742068_1244 (size=12001) 2024-11-11T07:30:22,606 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310282605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310282605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,607 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310282615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,619 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310282615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310282914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310282915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,920 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310282925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:22,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310282925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:22,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:22,941 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/5fe4c092e3214d2e98fbdac2a6db4ba1 2024-11-11T07:30:22,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/0bc0a3f2befb477ca9d02e8b9eaf2e16 is 50, key is test_row_0/C:col10/1731310221105/Put/seqid=0 2024-11-11T07:30:22,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742069_1245 (size=12001) 2024-11-11T07:30:23,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:23,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310283294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:23,298 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_1' in 385162ecea32f0e101e7709026aea910 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:23,364 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=124 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/0bc0a3f2befb477ca9d02e8b9eaf2e16 2024-11-11T07:30:23,370 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/6e626442fbf04bb7b2f0533c94f2e672 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/6e626442fbf04bb7b2f0533c94f2e672 2024-11-11T07:30:23,376 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/6e626442fbf04bb7b2f0533c94f2e672, entries=150, sequenceid=124, filesize=11.7 K 2024-11-11T07:30:23,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/5fe4c092e3214d2e98fbdac2a6db4ba1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5fe4c092e3214d2e98fbdac2a6db4ba1 2024-11-11T07:30:23,383 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5fe4c092e3214d2e98fbdac2a6db4ba1, entries=150, sequenceid=124, filesize=11.7 K 2024-11-11T07:30:23,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/0bc0a3f2befb477ca9d02e8b9eaf2e16 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/0bc0a3f2befb477ca9d02e8b9eaf2e16 2024-11-11T07:30:23,391 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/0bc0a3f2befb477ca9d02e8b9eaf2e16, entries=150, sequenceid=124, filesize=11.7 K 2024-11-11T07:30:23,392 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=46.96 KB/48090 for 385162ecea32f0e101e7709026aea910 in 1395ms, sequenceid=124, compaction requested=true 2024-11-11T07:30:23,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:23,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:23,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-11T07:30:23,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-11T07:30:23,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-11T07:30:23,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0150 sec 2024-11-11T07:30:23,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.0250 sec 2024-11-11T07:30:23,428 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T07:30:23,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:23,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:23,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:23,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:23,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:23,428 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:23,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:23,440 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/0652d6ea098749c1adf34451782cf190 is 50, key is test_row_0/A:col10/1731310222285/Put/seqid=0 2024-11-11T07:30:23,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742070_1246 (size=12147) 2024-11-11T07:30:23,464 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/0652d6ea098749c1adf34451782cf190 2024-11-11T07:30:23,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/83176f4f34bf495c9d1cf2dbf47ea9fb is 50, key is test_row_0/B:col10/1731310222285/Put/seqid=0 2024-11-11T07:30:23,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310223456, max timestamp=1731310223484Num uniques 859; 2024-11-11T07:30:23,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:30:23,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310223456, max timestamp=1731310223485Num uniques 859; 2024-11-11T07:30:23,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310223456, max timestamp=1731310223485Num uniques 859; 2024-11-11T07:30:23,488 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:30:23,489 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:30:23,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742071_1247 (size=9757) 2024-11-11T07:30:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-11T07:30:23,525 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:23,525 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:23,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-11T07:30:23,528 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:23,529 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:23,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:23,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T07:30:23,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:23,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310283537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:23,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:23,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310283537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:23,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:23,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310283546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:23,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:23,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:23,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310283547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:23,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,551 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:23,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,551 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:23,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T07:30:23,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:23,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310283654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,659 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:23,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310283655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:23,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:23,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310283655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:23,660 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:23,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310283655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,661 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:23,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T07:30:23,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:23,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:23,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:23,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:23,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:23,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:23,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T07:30:23,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:23,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:23,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:23,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:23,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:23,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:23,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T07:30:23,868 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:23,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310283865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:23,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310283865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:23,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310283865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,875 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:23,874 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:23,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310283865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:23,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:23,909 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/83176f4f34bf495c9d1cf2dbf47ea9fb 2024-11-11T07:30:23,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/601b092f716940aa886e5784a3004154 is 50, key is test_row_0/C:col10/1731310222285/Put/seqid=0 2024-11-11T07:30:23,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742072_1248 (size=9757) 2024-11-11T07:30:23,958 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/601b092f716940aa886e5784a3004154 2024-11-11T07:30:23,972 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/0652d6ea098749c1adf34451782cf190 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/0652d6ea098749c1adf34451782cf190 2024-11-11T07:30:23,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/0652d6ea098749c1adf34451782cf190, entries=150, sequenceid=135, filesize=11.9 K 2024-11-11T07:30:23,980 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/83176f4f34bf495c9d1cf2dbf47ea9fb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/83176f4f34bf495c9d1cf2dbf47ea9fb 2024-11-11T07:30:23,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T07:30:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:23,989 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/83176f4f34bf495c9d1cf2dbf47ea9fb, entries=100, sequenceid=135, filesize=9.5 K 2024-11-11T07:30:23,989 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:23,989 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:23,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:23,992 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/601b092f716940aa886e5784a3004154 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/601b092f716940aa886e5784a3004154 2024-11-11T07:30:24,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/601b092f716940aa886e5784a3004154, entries=100, sequenceid=135, filesize=9.5 K 2024-11-11T07:30:24,001 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 385162ecea32f0e101e7709026aea910 in 574ms, sequenceid=135, compaction requested=true 2024-11-11T07:30:24,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:24,001 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:24,007 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52933 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:24,007 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/A is initiating minor compaction (all files) 2024-11-11T07:30:24,007 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/A in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:24,007 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/64614399dfee4662965bc74dce12c5ff, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/a195c28df8494a568744efc561dc1d96, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/6e626442fbf04bb7b2f0533c94f2e672, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/0652d6ea098749c1adf34451782cf190] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=51.7 K 2024-11-11T07:30:24,008 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64614399dfee4662965bc74dce12c5ff, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731310218535 2024-11-11T07:30:24,008 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:24,008 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting a195c28df8494a568744efc561dc1d96, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731310219773 2024-11-11T07:30:24,009 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e626442fbf04bb7b2f0533c94f2e672, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731310221083 2024-11-11T07:30:24,010 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0652d6ea098749c1adf34451782cf190, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731310222280 2024-11-11T07:30:24,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:24,012 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:24,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:24,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:24,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:24,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:24,024 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45863 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:24,024 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/B is initiating minor compaction (all files) 2024-11-11T07:30:24,024 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/B in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:24,024 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/7845ea38663546bfa442860a0a784dfd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/7ed42671330e49c6bc8562780c6c563c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5fe4c092e3214d2e98fbdac2a6db4ba1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/83176f4f34bf495c9d1cf2dbf47ea9fb] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=44.8 K 2024-11-11T07:30:24,025 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 7845ea38663546bfa442860a0a784dfd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731310218535 2024-11-11T07:30:24,030 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ed42671330e49c6bc8562780c6c563c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731310220946 2024-11-11T07:30:24,035 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 5fe4c092e3214d2e98fbdac2a6db4ba1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731310221083 2024-11-11T07:30:24,051 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 83176f4f34bf495c9d1cf2dbf47ea9fb, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731310222285 2024-11-11T07:30:24,105 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#A#compaction#207 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:24,106 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/6975f93300a0450b89bac20cd0f365a4 is 50, key is test_row_0/A:col10/1731310222285/Put/seqid=0 2024-11-11T07:30:24,134 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#B#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:24,134 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/b900c33dc2b24dd397cae156489293e1 is 50, key is test_row_0/B:col10/1731310222285/Put/seqid=0 2024-11-11T07:30:24,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-11T07:30:24,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:24,151 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-11T07:30:24,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:24,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:24,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:24,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:24,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:24,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:24,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T07:30:24,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742073_1249 (size=12341) 2024-11-11T07:30:24,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/3237a58875854b78b7321266058d7425 is 50, key is test_row_0/A:col10/1731310223544/Put/seqid=0 2024-11-11T07:30:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:24,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:24,203 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/6975f93300a0450b89bac20cd0f365a4 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/6975f93300a0450b89bac20cd0f365a4 2024-11-11T07:30:24,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742074_1250 (size=12341) 2024-11-11T07:30:24,211 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/b900c33dc2b24dd397cae156489293e1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b900c33dc2b24dd397cae156489293e1 2024-11-11T07:30:24,211 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 385162ecea32f0e101e7709026aea910/A of 385162ecea32f0e101e7709026aea910 into 6975f93300a0450b89bac20cd0f365a4(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:24,211 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:24,211 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/A, priority=12, startTime=1731310224001; duration=0sec 2024-11-11T07:30:24,211 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:24,211 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:A 2024-11-11T07:30:24,211 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:24,214 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 45863 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:24,214 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/C is initiating minor compaction (all files) 2024-11-11T07:30:24,214 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/C in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:24,215 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/984395307849466188682aa9834674dd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/c0d053258c42402d9694b721dc7c5bf9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/0bc0a3f2befb477ca9d02e8b9eaf2e16, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/601b092f716940aa886e5784a3004154] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=44.8 K 2024-11-11T07:30:24,216 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 984395307849466188682aa9834674dd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731310218535 2024-11-11T07:30:24,216 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0d053258c42402d9694b721dc7c5bf9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731310220946 2024-11-11T07:30:24,216 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc0a3f2befb477ca9d02e8b9eaf2e16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=124, earliestPutTs=1731310221083 2024-11-11T07:30:24,217 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 601b092f716940aa886e5784a3004154, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731310222285 2024-11-11T07:30:24,221 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 385162ecea32f0e101e7709026aea910/B of 385162ecea32f0e101e7709026aea910 into b900c33dc2b24dd397cae156489293e1(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:24,221 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:24,221 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/B, priority=12, startTime=1731310224012; duration=0sec 2024-11-11T07:30:24,221 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:24,221 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:B 2024-11-11T07:30:24,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310284219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,231 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,231 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310284221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310284221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,232 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310284231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742075_1251 (size=12151) 2024-11-11T07:30:24,244 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/3237a58875854b78b7321266058d7425 2024-11-11T07:30:24,254 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#C#compaction#210 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:24,255 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/de1fcaa86eb24a7c99a7148a1d2c52da is 50, key is test_row_0/C:col10/1731310222285/Put/seqid=0 2024-11-11T07:30:24,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/1b6d6ecdfd7c477b878202c73c6d667f is 50, key is test_row_0/B:col10/1731310223544/Put/seqid=0 2024-11-11T07:30:24,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742076_1252 (size=12341) 2024-11-11T07:30:24,298 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/de1fcaa86eb24a7c99a7148a1d2c52da as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/de1fcaa86eb24a7c99a7148a1d2c52da 2024-11-11T07:30:24,306 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 385162ecea32f0e101e7709026aea910/C of 385162ecea32f0e101e7709026aea910 into de1fcaa86eb24a7c99a7148a1d2c52da(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:24,306 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:24,306 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/C, priority=12, startTime=1731310224012; duration=0sec 2024-11-11T07:30:24,306 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:24,306 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:C 2024-11-11T07:30:24,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742077_1253 (size=12151) 2024-11-11T07:30:24,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310284336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310284336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310284336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,341 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,351 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/1b6d6ecdfd7c477b878202c73c6d667f 2024-11-11T07:30:24,360 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310284354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,362 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/642ee7782f0244af8e25eed4aeb929b3 is 50, key is test_row_0/C:col10/1731310223544/Put/seqid=0 2024-11-11T07:30:24,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742078_1254 (size=12151) 2024-11-11T07:30:24,397 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=171 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/642ee7782f0244af8e25eed4aeb929b3 2024-11-11T07:30:24,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/3237a58875854b78b7321266058d7425 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/3237a58875854b78b7321266058d7425 2024-11-11T07:30:24,407 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/3237a58875854b78b7321266058d7425, entries=150, sequenceid=171, filesize=11.9 K 2024-11-11T07:30:24,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/1b6d6ecdfd7c477b878202c73c6d667f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/1b6d6ecdfd7c477b878202c73c6d667f 2024-11-11T07:30:24,414 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/1b6d6ecdfd7c477b878202c73c6d667f, entries=150, sequenceid=171, filesize=11.9 K 2024-11-11T07:30:24,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/642ee7782f0244af8e25eed4aeb929b3 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/642ee7782f0244af8e25eed4aeb929b3 2024-11-11T07:30:24,421 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/642ee7782f0244af8e25eed4aeb929b3, entries=150, sequenceid=171, filesize=11.9 K 2024-11-11T07:30:24,423 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 385162ecea32f0e101e7709026aea910 in 272ms, sequenceid=171, compaction requested=false 2024-11-11T07:30:24,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:24,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:24,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-11T07:30:24,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-11T07:30:24,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-11T07:30:24,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 896 msec 2024-11-11T07:30:24,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 904 msec 2024-11-11T07:30:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:24,547 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T07:30:24,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:24,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:24,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:24,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:24,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:24,548 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:24,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/2ab6035de7a346dd88b04aba76f648cc is 50, key is test_row_0/A:col10/1731310224228/Put/seqid=0 2024-11-11T07:30:24,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310284593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310284595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,605 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310284597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310284597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,606 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,606 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742079_1255 (size=12151) 2024-11-11T07:30:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-11T07:30:24,682 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:24,683 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:24,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-11T07:30:24,685 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:24,685 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:24,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T07:30:24,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310284715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310284715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310284715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,719 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,720 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310284716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T07:30:24,839 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T07:30:24,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:24,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:24,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:24,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:24,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:24,891 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-11T07:30:24,891 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-11T07:30:24,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310284924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310284924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310284925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,930 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,938 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:24,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310284935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:24,938 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:24,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T07:30:24,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:24,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:24,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:24,993 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:24,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:24,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T07:30:25,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/2ab6035de7a346dd88b04aba76f648cc 2024-11-11T07:30:25,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/aaeccd367e26437abc270e4a301fb403 is 50, key is test_row_0/B:col10/1731310224228/Put/seqid=0 2024-11-11T07:30:25,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742080_1256 (size=12151) 2024-11-11T07:30:25,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T07:30:25,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:25,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:25,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:25,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,244 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:25,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310285239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,245 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:25,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310285239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,245 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:25,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:25,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310285239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:25,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,246 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:25,253 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:25,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310285247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:25,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T07:30:25,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:25,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:25,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:25,300 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T07:30:25,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:25,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310285315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:25,320 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_1' in 385162ecea32f0e101e7709026aea910 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4207 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:25,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,320 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:25,458 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T07:30:25,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:25,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:25,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:25,459 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/aaeccd367e26437abc270e4a301fb403 2024-11-11T07:30:25,532 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/b13cd70f417846ab8b6d3ed714fe44b8 is 50, key is test_row_0/C:col10/1731310224228/Put/seqid=0 2024-11-11T07:30:25,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742081_1257 (size=12151) 2024-11-11T07:30:25,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T07:30:25,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:25,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:25,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:25,615 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:25,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310285755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310285755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:25,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:25,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310285756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:25,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,762 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:25,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:25,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310285764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:25,766 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:25,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T07:30:25,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:25,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:25,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:25,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T07:30:25,952 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T07:30:25,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:25,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:25,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:25,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:25,954 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=187 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/b13cd70f417846ab8b6d3ed714fe44b8 2024-11-11T07:30:25,986 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/2ab6035de7a346dd88b04aba76f648cc as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/2ab6035de7a346dd88b04aba76f648cc 2024-11-11T07:30:26,000 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/2ab6035de7a346dd88b04aba76f648cc, entries=150, sequenceid=187, filesize=11.9 K 2024-11-11T07:30:26,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/aaeccd367e26437abc270e4a301fb403 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/aaeccd367e26437abc270e4a301fb403 2024-11-11T07:30:26,007 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/aaeccd367e26437abc270e4a301fb403, entries=150, sequenceid=187, filesize=11.9 K 2024-11-11T07:30:26,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/b13cd70f417846ab8b6d3ed714fe44b8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/b13cd70f417846ab8b6d3ed714fe44b8 2024-11-11T07:30:26,026 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/b13cd70f417846ab8b6d3ed714fe44b8, entries=150, sequenceid=187, filesize=11.9 K 2024-11-11T07:30:26,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 385162ecea32f0e101e7709026aea910 in 1484ms, sequenceid=187, compaction requested=true 2024-11-11T07:30:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:30:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:26,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T07:30:26,032 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:26,033 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:26,033 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/A is initiating minor compaction (all files) 2024-11-11T07:30:26,033 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/A in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:26,033 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/6975f93300a0450b89bac20cd0f365a4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/3237a58875854b78b7321266058d7425, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/2ab6035de7a346dd88b04aba76f648cc] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=35.8 K 2024-11-11T07:30:26,034 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 6975f93300a0450b89bac20cd0f365a4, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731310221097 2024-11-11T07:30:26,034 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:26,034 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 3237a58875854b78b7321266058d7425, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731310223536 2024-11-11T07:30:26,036 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ab6035de7a346dd88b04aba76f648cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731310224228 2024-11-11T07:30:26,045 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#A#compaction#216 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:26,045 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/d0ec91f49efe4990ab8b6f9a4d25ccf5 is 50, key is test_row_0/A:col10/1731310224228/Put/seqid=0 2024-11-11T07:30:26,050 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:26,051 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/C is initiating minor compaction (all files) 2024-11-11T07:30:26,051 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/C in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:26,051 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/de1fcaa86eb24a7c99a7148a1d2c52da, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/642ee7782f0244af8e25eed4aeb929b3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/b13cd70f417846ab8b6d3ed714fe44b8] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=35.8 K 2024-11-11T07:30:26,052 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting de1fcaa86eb24a7c99a7148a1d2c52da, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731310221097 2024-11-11T07:30:26,053 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 642ee7782f0244af8e25eed4aeb929b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731310223536 2024-11-11T07:30:26,053 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting b13cd70f417846ab8b6d3ed714fe44b8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731310224228 2024-11-11T07:30:26,069 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#C#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:26,069 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/9de8fdd8647b4630b5aa4f4f5b8d7154 is 50, key is test_row_0/C:col10/1731310224228/Put/seqid=0 2024-11-11T07:30:26,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742082_1258 (size=12493) 2024-11-11T07:30:26,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742083_1259 (size=12493) 2024-11-11T07:30:26,107 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-11T07:30:26,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:26,111 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T07:30:26,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:26,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:26,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:26,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:26,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:26,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:26,119 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/9de8fdd8647b4630b5aa4f4f5b8d7154 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/9de8fdd8647b4630b5aa4f4f5b8d7154 2024-11-11T07:30:26,131 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/C of 385162ecea32f0e101e7709026aea910 into 9de8fdd8647b4630b5aa4f4f5b8d7154(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:26,131 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:26,131 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/C, priority=13, startTime=1731310226031; duration=0sec 2024-11-11T07:30:26,131 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:26,131 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:C 2024-11-11T07:30:26,131 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:26,133 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:26,133 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/B is initiating minor compaction (all files) 2024-11-11T07:30:26,133 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/B in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:26,133 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b900c33dc2b24dd397cae156489293e1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/1b6d6ecdfd7c477b878202c73c6d667f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/aaeccd367e26437abc270e4a301fb403] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=35.8 K 2024-11-11T07:30:26,134 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting b900c33dc2b24dd397cae156489293e1, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731310221097 2024-11-11T07:30:26,134 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b6d6ecdfd7c477b878202c73c6d667f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=171, earliestPutTs=1731310223536 2024-11-11T07:30:26,135 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting aaeccd367e26437abc270e4a301fb403, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731310224228 2024-11-11T07:30:26,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/e6583b69320e4191956400c86d6e6157 is 50, key is test_row_0/A:col10/1731310224595/Put/seqid=0 2024-11-11T07:30:26,153 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#B#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:26,153 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/4e7e45236a4744059ace6a8fae865c5a is 50, key is test_row_0/B:col10/1731310224228/Put/seqid=0 2024-11-11T07:30:26,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742084_1260 (size=12151) 2024-11-11T07:30:26,211 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/e6583b69320e4191956400c86d6e6157 2024-11-11T07:30:26,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742085_1261 (size=12493) 2024-11-11T07:30:26,226 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/4e7e45236a4744059ace6a8fae865c5a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/4e7e45236a4744059ace6a8fae865c5a 2024-11-11T07:30:26,235 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/B of 385162ecea32f0e101e7709026aea910 into 4e7e45236a4744059ace6a8fae865c5a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:26,235 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:26,235 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/B, priority=13, startTime=1731310226031; duration=0sec 2024-11-11T07:30:26,235 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:26,235 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:B 2024-11-11T07:30:26,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/bdb907ddf85144db9da87c120140b710 is 50, key is test_row_0/B:col10/1731310224595/Put/seqid=0 2024-11-11T07:30:26,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742086_1262 (size=12151) 2024-11-11T07:30:26,279 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/bdb907ddf85144db9da87c120140b710 2024-11-11T07:30:26,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/f570bea7ef1844cd99425ed4ea80e012 is 50, key is test_row_0/C:col10/1731310224595/Put/seqid=0 2024-11-11T07:30:26,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742087_1263 (size=12151) 2024-11-11T07:30:26,499 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/d0ec91f49efe4990ab8b6f9a4d25ccf5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/d0ec91f49efe4990ab8b6f9a4d25ccf5 2024-11-11T07:30:26,505 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/A of 385162ecea32f0e101e7709026aea910 into d0ec91f49efe4990ab8b6f9a4d25ccf5(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:26,505 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:26,505 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/A, priority=13, startTime=1731310226031; duration=0sec 2024-11-11T07:30:26,506 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:26,506 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:A 2024-11-11T07:30:26,750 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/f570bea7ef1844cd99425ed4ea80e012 2024-11-11T07:30:26,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/e6583b69320e4191956400c86d6e6157 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/e6583b69320e4191956400c86d6e6157 2024-11-11T07:30:26,772 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/e6583b69320e4191956400c86d6e6157, entries=150, sequenceid=206, filesize=11.9 K 2024-11-11T07:30:26,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/bdb907ddf85144db9da87c120140b710 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/bdb907ddf85144db9da87c120140b710 2024-11-11T07:30:26,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:26,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:26,786 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/bdb907ddf85144db9da87c120140b710, entries=150, sequenceid=206, filesize=11.9 K 2024-11-11T07:30:26,786 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-11T07:30:26,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/f570bea7ef1844cd99425ed4ea80e012 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/f570bea7ef1844cd99425ed4ea80e012 2024-11-11T07:30:26,804 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/f570bea7ef1844cd99425ed4ea80e012, entries=150, sequenceid=206, filesize=11.9 K 2024-11-11T07:30:26,805 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 385162ecea32f0e101e7709026aea910 in 695ms, sequenceid=206, compaction requested=false 2024-11-11T07:30:26,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:26,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:26,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-11T07:30:26,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-11T07:30:26,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:26,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T07:30:26,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:26,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:26,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:26,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:26,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:26,814 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:26,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-11T07:30:26,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1320 sec 2024-11-11T07:30:26,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.1390 sec 2024-11-11T07:30:26,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9fcb95b6ec984115b10fefe4cf5bc615 is 50, key is test_row_0/A:col10/1731310226800/Put/seqid=0 2024-11-11T07:30:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-11T07:30:26,845 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:26,846 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:26,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-11T07:30:26,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T07:30:26,855 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:26,856 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:26,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:26,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310286858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:26,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:26,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:26,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:26,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310286859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:26,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:26,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:26,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310286860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:26,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742088_1264 (size=14541) 2024-11-11T07:30:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:26,871 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:26,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:26,872 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:26,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:26,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310286867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:26,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:26,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:26,874 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:26,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9fcb95b6ec984115b10fefe4cf5bc615 2024-11-11T07:30:26,888 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/b1073c6ff2e84f06965601c731bd6805 is 50, key is test_row_0/B:col10/1731310226800/Put/seqid=0 2024-11-11T07:30:26,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742089_1265 (size=12151) 2024-11-11T07:30:26,933 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/b1073c6ff2e84f06965601c731bd6805 2024-11-11T07:30:26,952 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/34109cddc51e461698efc030e842c452 is 50, key is test_row_0/C:col10/1731310226800/Put/seqid=0 2024-11-11T07:30:26,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T07:30:26,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:26,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310286975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:26,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310286975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:26,978 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:26,978 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310286975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:26,979 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:26,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:26,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310286984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:26,988 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:26,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742090_1266 (size=12151) 2024-11-11T07:30:27,001 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/34109cddc51e461698efc030e842c452 2024-11-11T07:30:27,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T07:30:27,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:27,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:27,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:27,009 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:27,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:27,010 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9fcb95b6ec984115b10fefe4cf5bc615 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9fcb95b6ec984115b10fefe4cf5bc615 2024-11-11T07:30:27,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:27,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9fcb95b6ec984115b10fefe4cf5bc615, entries=200, sequenceid=225, filesize=14.2 K 2024-11-11T07:30:27,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/b1073c6ff2e84f06965601c731bd6805 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b1073c6ff2e84f06965601c731bd6805 2024-11-11T07:30:27,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b1073c6ff2e84f06965601c731bd6805, entries=150, sequenceid=225, filesize=11.9 K 2024-11-11T07:30:27,027 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/34109cddc51e461698efc030e842c452 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/34109cddc51e461698efc030e842c452 2024-11-11T07:30:27,033 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/34109cddc51e461698efc030e842c452, entries=150, sequenceid=225, filesize=11.9 K 2024-11-11T07:30:27,035 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 385162ecea32f0e101e7709026aea910 in 225ms, sequenceid=225, compaction requested=true 2024-11-11T07:30:27,036 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:27,036 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:27,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:27,037 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:27,037 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/A is initiating minor compaction (all files) 2024-11-11T07:30:27,037 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/A in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:27,037 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/d0ec91f49efe4990ab8b6f9a4d25ccf5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/e6583b69320e4191956400c86d6e6157, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9fcb95b6ec984115b10fefe4cf5bc615] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=38.3 K 2024-11-11T07:30:27,038 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0ec91f49efe4990ab8b6f9a4d25ccf5, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731310224228 2024-11-11T07:30:27,036 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:27,038 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:27,038 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:27,040 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting e6583b69320e4191956400c86d6e6157, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731310224593 2024-11-11T07:30:27,040 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fcb95b6ec984115b10fefe4cf5bc615, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731310226798 2024-11-11T07:30:27,042 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:27,042 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/B is initiating minor compaction (all files) 2024-11-11T07:30:27,042 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/B in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:27,042 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/4e7e45236a4744059ace6a8fae865c5a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/bdb907ddf85144db9da87c120140b710, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b1073c6ff2e84f06965601c731bd6805] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=35.9 K 2024-11-11T07:30:27,043 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e7e45236a4744059ace6a8fae865c5a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731310224228 2024-11-11T07:30:27,044 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting bdb907ddf85144db9da87c120140b710, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731310224593 2024-11-11T07:30:27,045 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting b1073c6ff2e84f06965601c731bd6805, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731310226798 2024-11-11T07:30:27,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:27,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:27,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:27,065 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#A#compaction#225 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:27,066 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/11becc0501dc49b58a9506e4ba3deb68 is 50, key is test_row_0/A:col10/1731310226800/Put/seqid=0 2024-11-11T07:30:27,078 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#B#compaction#226 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:27,078 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/fa4cdc36e95c4539b793fee9c7e04827 is 50, key is test_row_0/B:col10/1731310226800/Put/seqid=0 2024-11-11T07:30:27,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742091_1267 (size=12595) 2024-11-11T07:30:27,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742092_1268 (size=12595) 2024-11-11T07:30:27,126 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/fa4cdc36e95c4539b793fee9c7e04827 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/fa4cdc36e95c4539b793fee9c7e04827 2024-11-11T07:30:27,134 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/11becc0501dc49b58a9506e4ba3deb68 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/11becc0501dc49b58a9506e4ba3deb68 2024-11-11T07:30:27,144 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/A of 385162ecea32f0e101e7709026aea910 into 11becc0501dc49b58a9506e4ba3deb68(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:27,144 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:27,144 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/A, priority=13, startTime=1731310227036; duration=0sec 2024-11-11T07:30:27,144 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:27,144 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:A 2024-11-11T07:30:27,144 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:27,145 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:27,146 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/C is initiating minor compaction (all files) 2024-11-11T07:30:27,146 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/C in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:27,146 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/9de8fdd8647b4630b5aa4f4f5b8d7154, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/f570bea7ef1844cd99425ed4ea80e012, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/34109cddc51e461698efc030e842c452] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=35.9 K 2024-11-11T07:30:27,146 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9de8fdd8647b4630b5aa4f4f5b8d7154, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=187, earliestPutTs=1731310224228 2024-11-11T07:30:27,146 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting f570bea7ef1844cd99425ed4ea80e012, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731310224593 2024-11-11T07:30:27,147 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 34109cddc51e461698efc030e842c452, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731310226798 2024-11-11T07:30:27,147 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/B of 385162ecea32f0e101e7709026aea910 into fa4cdc36e95c4539b793fee9c7e04827(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:27,147 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:27,148 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/B, priority=13, startTime=1731310227038; duration=0sec 2024-11-11T07:30:27,148 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:27,148 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:B 2024-11-11T07:30:27,154 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#C#compaction#227 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:27,154 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/96602b52926349689d943bf3c735589d is 50, key is test_row_0/C:col10/1731310226800/Put/seqid=0 2024-11-11T07:30:27,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742093_1269 (size=12595) 2024-11-11T07:30:27,167 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/96602b52926349689d943bf3c735589d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/96602b52926349689d943bf3c735589d 2024-11-11T07:30:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T07:30:27,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-11T07:30:27,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:27,173 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T07:30:27,174 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/C of 385162ecea32f0e101e7709026aea910 into 96602b52926349689d943bf3c735589d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:27,174 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:27,174 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/C, priority=13, startTime=1731310227059; duration=0sec 2024-11-11T07:30:27,174 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:27,175 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:C 2024-11-11T07:30:27,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:27,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:27,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:27,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:27,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:27,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:27,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9fc4f6ed4431419ba15bc9b7ee639736 is 50, key is test_row_0/A:col10/1731310226853/Put/seqid=0 2024-11-11T07:30:27,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:27,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:27,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:27,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310287213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:27,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,224 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:27,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:27,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310287215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:27,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:27,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310287217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:27,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742094_1270 (size=12151) 2024-11-11T07:30:27,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:27,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310287224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:27,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310287338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:27,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310287338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:27,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,344 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:27,344 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:27,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310287339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:27,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,345 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:27,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,345 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:27,351 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:27,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310287345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:27,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,352 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:27,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T07:30:27,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310287554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:27,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:27,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310287555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:27,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310287555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:27,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:27,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:27,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310287566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:27,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:27,633 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9fc4f6ed4431419ba15bc9b7ee639736 2024-11-11T07:30:27,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/fa19a709baa247648b4ff654b3f76fcb is 50, key is test_row_0/B:col10/1731310226853/Put/seqid=0 2024-11-11T07:30:27,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742095_1271 (size=12151) 2024-11-11T07:30:27,672 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/fa19a709baa247648b4ff654b3f76fcb 2024-11-11T07:30:27,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/a3bc54b3f64748999a4f99192dec1ed0 is 50, key is test_row_0/C:col10/1731310226853/Put/seqid=0 2024-11-11T07:30:27,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742096_1272 (size=12151) 2024-11-11T07:30:27,720 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/a3bc54b3f64748999a4f99192dec1ed0 2024-11-11T07:30:27,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9fc4f6ed4431419ba15bc9b7ee639736 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9fc4f6ed4431419ba15bc9b7ee639736 2024-11-11T07:30:27,737 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9fc4f6ed4431419ba15bc9b7ee639736, entries=150, sequenceid=244, filesize=11.9 K 2024-11-11T07:30:27,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/fa19a709baa247648b4ff654b3f76fcb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/fa19a709baa247648b4ff654b3f76fcb 2024-11-11T07:30:27,746 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/fa19a709baa247648b4ff654b3f76fcb, entries=150, sequenceid=244, filesize=11.9 K 2024-11-11T07:30:27,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/a3bc54b3f64748999a4f99192dec1ed0 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a3bc54b3f64748999a4f99192dec1ed0 2024-11-11T07:30:27,752 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a3bc54b3f64748999a4f99192dec1ed0, entries=150, sequenceid=244, filesize=11.9 K 2024-11-11T07:30:27,753 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 385162ecea32f0e101e7709026aea910 in 580ms, sequenceid=244, compaction requested=false 2024-11-11T07:30:27,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:27,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:27,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-11T07:30:27,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-11T07:30:27,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-11T07:30:27,759 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 900 msec 2024-11-11T07:30:27,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 914 msec 2024-11-11T07:30:27,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:27,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T07:30:27,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:27,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:27,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:27,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:27,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:27,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:27,882 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/013fb53ecb39425a87633c97ace8fc9d is 50, key is test_row_0/A:col10/1731310227868/Put/seqid=0 2024-11-11T07:30:27,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:27,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310287895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:27,910 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:27,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310287904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:27,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:27,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310287905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:27,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:27,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310287905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:27,913 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:27,920 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742097_1273 (size=19571) 2024-11-11T07:30:27,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/013fb53ecb39425a87633c97ace8fc9d 2024-11-11T07:30:27,933 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/24661ec424ec4f738aec6e741411ad35 is 50, key is test_row_0/B:col10/1731310227868/Put/seqid=0 2024-11-11T07:30:27,979 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742098_1274 (size=12251) 2024-11-11T07:30:27,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-11T07:30:27,985 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:27,985 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-11T07:30:27,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/24661ec424ec4f738aec6e741411ad35 2024-11-11T07:30:27,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T07:30:27,988 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:27,989 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:27,989 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:28,000 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/3822e797d86f410496cd154cc0fb6ce8 is 50, key is test_row_0/C:col10/1731310227868/Put/seqid=0 2024-11-11T07:30:28,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742099_1275 (size=12251) 2024-11-11T07:30:28,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/3822e797d86f410496cd154cc0fb6ce8 2024-11-11T07:30:28,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,017 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310288014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310288014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,018 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,019 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/013fb53ecb39425a87633c97ace8fc9d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/013fb53ecb39425a87633c97ace8fc9d 2024-11-11T07:30:28,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310288025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,030 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310288025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/013fb53ecb39425a87633c97ace8fc9d, entries=300, sequenceid=263, filesize=19.1 K 2024-11-11T07:30:28,038 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/24661ec424ec4f738aec6e741411ad35 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/24661ec424ec4f738aec6e741411ad35 2024-11-11T07:30:28,043 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/24661ec424ec4f738aec6e741411ad35, entries=150, sequenceid=263, filesize=12.0 K 2024-11-11T07:30:28,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/3822e797d86f410496cd154cc0fb6ce8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/3822e797d86f410496cd154cc0fb6ce8 2024-11-11T07:30:28,049 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/3822e797d86f410496cd154cc0fb6ce8, entries=150, sequenceid=263, filesize=12.0 K 2024-11-11T07:30:28,051 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 385162ecea32f0e101e7709026aea910 in 183ms, sequenceid=263, compaction requested=true 2024-11-11T07:30:28,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:28,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:28,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:28,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:28,051 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:28,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:28,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:28,051 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:30:28,051 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:28,054 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36997 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:28,054 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 44317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:28,054 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/B is initiating minor compaction (all files) 2024-11-11T07:30:28,054 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/A is initiating minor compaction (all files) 2024-11-11T07:30:28,054 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/A in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:28,054 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/B in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:28,054 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/11becc0501dc49b58a9506e4ba3deb68, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9fc4f6ed4431419ba15bc9b7ee639736, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/013fb53ecb39425a87633c97ace8fc9d] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=43.3 K 2024-11-11T07:30:28,054 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/fa4cdc36e95c4539b793fee9c7e04827, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/fa19a709baa247648b4ff654b3f76fcb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/24661ec424ec4f738aec6e741411ad35] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=36.1 K 2024-11-11T07:30:28,055 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11becc0501dc49b58a9506e4ba3deb68, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731310226798 2024-11-11T07:30:28,055 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting fa4cdc36e95c4539b793fee9c7e04827, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731310226798 2024-11-11T07:30:28,055 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting fa19a709baa247648b4ff654b3f76fcb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731310226853 2024-11-11T07:30:28,055 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9fc4f6ed4431419ba15bc9b7ee639736, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731310226853 2024-11-11T07:30:28,056 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 24661ec424ec4f738aec6e741411ad35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731310227208 2024-11-11T07:30:28,056 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 013fb53ecb39425a87633c97ace8fc9d, keycount=300, bloomtype=ROW, size=19.1 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731310227208 2024-11-11T07:30:28,069 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#B#compaction#234 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:28,069 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/4412a911f5634d18834ff3cbc5e6f292 is 50, key is test_row_0/B:col10/1731310227868/Put/seqid=0 2024-11-11T07:30:28,074 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#A#compaction#235 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:28,074 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/6850c79d91fb44beb626e0de9303b4a8 is 50, key is test_row_0/A:col10/1731310227868/Put/seqid=0 2024-11-11T07:30:28,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T07:30:28,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742101_1277 (size=12797) 2024-11-11T07:30:28,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742100_1276 (size=12797) 2024-11-11T07:30:28,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-11T07:30:28,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:28,147 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T07:30:28,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:28,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:28,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:28,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:28,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:28,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:28,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9544dc96841943608140977ab3060c93 is 50, key is test_row_0/A:col10/1731310227903/Put/seqid=0 2024-11-11T07:30:28,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742102_1278 (size=12301) 2024-11-11T07:30:28,211 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9544dc96841943608140977ab3060c93 2024-11-11T07:30:28,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:28,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:28,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/8ab5842218e34a6389bbeb0a7eeb00e2 is 50, key is test_row_0/B:col10/1731310227903/Put/seqid=0 2024-11-11T07:30:28,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742103_1279 (size=12301) 2024-11-11T07:30:28,251 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/8ab5842218e34a6389bbeb0a7eeb00e2 2024-11-11T07:30:28,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/7312ff592b084feab029e115d613a438 is 50, key is test_row_0/C:col10/1731310227903/Put/seqid=0 2024-11-11T07:30:28,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310288257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,267 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,274 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310288266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310288265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310288267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,275 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742104_1280 (size=12301) 2024-11-11T07:30:28,279 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=277 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/7312ff592b084feab029e115d613a438 2024-11-11T07:30:28,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9544dc96841943608140977ab3060c93 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9544dc96841943608140977ab3060c93 2024-11-11T07:30:28,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T07:30:28,348 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9544dc96841943608140977ab3060c93, entries=150, sequenceid=277, filesize=12.0 K 2024-11-11T07:30:28,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/8ab5842218e34a6389bbeb0a7eeb00e2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/8ab5842218e34a6389bbeb0a7eeb00e2 2024-11-11T07:30:28,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310288375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,380 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,384 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/8ab5842218e34a6389bbeb0a7eeb00e2, entries=150, sequenceid=277, filesize=12.0 K 2024-11-11T07:30:28,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/7312ff592b084feab029e115d613a438 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/7312ff592b084feab029e115d613a438 2024-11-11T07:30:28,392 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310288385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310288385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310288385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,393 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,394 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,395 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/7312ff592b084feab029e115d613a438, entries=150, sequenceid=277, filesize=12.0 K 2024-11-11T07:30:28,398 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 385162ecea32f0e101e7709026aea910 in 251ms, sequenceid=277, compaction requested=true 2024-11-11T07:30:28,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:28,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:28,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-11T07:30:28,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-11T07:30:28,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-11T07:30:28,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 410 msec 2024-11-11T07:30:28,408 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 420 msec 2024-11-11T07:30:28,575 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/4412a911f5634d18834ff3cbc5e6f292 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/4412a911f5634d18834ff3cbc5e6f292 2024-11-11T07:30:28,578 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/6850c79d91fb44beb626e0de9303b4a8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/6850c79d91fb44beb626e0de9303b4a8 2024-11-11T07:30:28,590 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T07:30:28,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:28,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:28,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:28,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:28,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:28,590 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:28,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:28,604 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/19edb9b1c33147bc8ea144fa15075f87 is 50, key is test_row_0/A:col10/1731310228265/Put/seqid=0 2024-11-11T07:30:28,611 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/B of 385162ecea32f0e101e7709026aea910 into 4412a911f5634d18834ff3cbc5e6f292(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:28,611 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:28,611 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/B, priority=13, startTime=1731310228051; duration=0sec 2024-11-11T07:30:28,612 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:28,612 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:B 2024-11-11T07:30:28,612 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:28,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-11T07:30:28,615 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:28,616 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:28,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-11T07:30:28,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T07:30:28,618 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:28,618 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49298 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:28,618 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/C is initiating minor compaction (all files) 2024-11-11T07:30:28,618 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/C in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:28,618 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/96602b52926349689d943bf3c735589d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a3bc54b3f64748999a4f99192dec1ed0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/3822e797d86f410496cd154cc0fb6ce8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/7312ff592b084feab029e115d613a438] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=48.1 K 2024-11-11T07:30:28,619 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 96602b52926349689d943bf3c735589d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731310226798 2024-11-11T07:30:28,619 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting a3bc54b3f64748999a4f99192dec1ed0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731310226853 2024-11-11T07:30:28,620 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 3822e797d86f410496cd154cc0fb6ce8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731310227208 2024-11-11T07:30:28,620 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 7312ff592b084feab029e115d613a438, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731310227872 2024-11-11T07:30:28,621 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/A of 385162ecea32f0e101e7709026aea910 into 6850c79d91fb44beb626e0de9303b4a8(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:28,621 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:28,621 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/A, priority=13, startTime=1731310228051; duration=0sec 2024-11-11T07:30:28,621 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:28,621 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:A 2024-11-11T07:30:28,622 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:28,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:28,636 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#C#compaction#240 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:28,637 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/0bdecc321b2a469d92fcb960a68b2989 is 50, key is test_row_0/C:col10/1731310227903/Put/seqid=0 2024-11-11T07:30:28,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310288626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,638 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742105_1281 (size=14741) 2024-11-11T07:30:28,644 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310288635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,646 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/19edb9b1c33147bc8ea144fa15075f87 2024-11-11T07:30:28,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310288637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310288635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,659 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,672 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/931f7d1409474051a0df72fba4e62ff5 is 50, key is test_row_0/B:col10/1731310228265/Put/seqid=0 2024-11-11T07:30:28,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742106_1282 (size=12881) 2024-11-11T07:30:28,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T07:30:28,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742107_1283 (size=12301) 2024-11-11T07:30:28,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/931f7d1409474051a0df72fba4e62ff5 2024-11-11T07:30:28,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310288744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,751 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310288754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,760 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,760 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310288754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,761 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/ddb417b8c43c49c48472142537247fa2 is 50, key is test_row_0/C:col10/1731310228265/Put/seqid=0 2024-11-11T07:30:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:28,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310288765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:28,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:28,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T07:30:28,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:28,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:28,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:28,783 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:28,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:28,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:28,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742108_1284 (size=12301) 2024-11-11T07:30:28,806 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/ddb417b8c43c49c48472142537247fa2 2024-11-11T07:30:28,811 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/19edb9b1c33147bc8ea144fa15075f87 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/19edb9b1c33147bc8ea144fa15075f87 2024-11-11T07:30:28,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/19edb9b1c33147bc8ea144fa15075f87, entries=200, sequenceid=296, filesize=14.4 K 2024-11-11T07:30:28,823 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/931f7d1409474051a0df72fba4e62ff5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/931f7d1409474051a0df72fba4e62ff5 2024-11-11T07:30:28,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/931f7d1409474051a0df72fba4e62ff5, entries=150, sequenceid=296, filesize=12.0 K 2024-11-11T07:30:28,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/ddb417b8c43c49c48472142537247fa2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/ddb417b8c43c49c48472142537247fa2 2024-11-11T07:30:28,848 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/ddb417b8c43c49c48472142537247fa2, entries=150, sequenceid=296, filesize=12.0 K 2024-11-11T07:30:28,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 385162ecea32f0e101e7709026aea910 in 259ms, sequenceid=296, compaction requested=true 2024-11-11T07:30:28,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:28,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:A, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:28,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:28,849 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:28,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:B, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:28,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:28,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:28,849 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:30:28,850 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39839 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:28,850 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/A is initiating minor compaction (all files) 2024-11-11T07:30:28,850 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/A in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:28,850 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/6850c79d91fb44beb626e0de9303b4a8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9544dc96841943608140977ab3060c93, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/19edb9b1c33147bc8ea144fa15075f87] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=38.9 K 2024-11-11T07:30:28,855 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6850c79d91fb44beb626e0de9303b4a8, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731310227208 2024-11-11T07:30:28,855 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9544dc96841943608140977ab3060c93, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731310227872 2024-11-11T07:30:28,856 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 19edb9b1c33147bc8ea144fa15075f87, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731310228262 2024-11-11T07:30:28,873 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#A#compaction#243 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:28,873 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/8a2ad4117fee46e98b46e5515ba11621 is 50, key is test_row_0/A:col10/1731310228265/Put/seqid=0 2024-11-11T07:30:28,930 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742109_1285 (size=12949) 2024-11-11T07:30:28,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T07:30:28,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-11T07:30:28,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:28,939 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T07:30:28,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:28,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:28,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:28,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:28,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:28,939 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:28,945 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/8a2ad4117fee46e98b46e5515ba11621 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/8a2ad4117fee46e98b46e5515ba11621 2024-11-11T07:30:28,952 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/A of 385162ecea32f0e101e7709026aea910 into 8a2ad4117fee46e98b46e5515ba11621(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:28,952 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:28,952 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/A, priority=13, startTime=1731310228849; duration=0sec 2024-11-11T07:30:28,952 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:30:28,952 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:A 2024-11-11T07:30:28,952 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 4 compacting, 1 eligible, 16 blocking 2024-11-11T07:30:28,953 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T07:30:28,953 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T07:30:28,953 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. because compaction request was cancelled 2024-11-11T07:30:28,953 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:C 2024-11-11T07:30:28,953 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:28,954 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:28,954 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/B is initiating minor compaction (all files) 2024-11-11T07:30:28,954 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/B in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:28,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9b6318133d324148abb4de9403a5b07d is 50, key is test_row_0/A:col10/1731310228635/Put/seqid=0 2024-11-11T07:30:28,957 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/4412a911f5634d18834ff3cbc5e6f292, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/8ab5842218e34a6389bbeb0a7eeb00e2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/931f7d1409474051a0df72fba4e62ff5] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=36.5 K 2024-11-11T07:30:28,958 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4412a911f5634d18834ff3cbc5e6f292, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731310227208 2024-11-11T07:30:28,958 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ab5842218e34a6389bbeb0a7eeb00e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731310227872 2024-11-11T07:30:28,959 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 931f7d1409474051a0df72fba4e62ff5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731310228262 2024-11-11T07:30:28,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:28,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:28,972 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#B#compaction#245 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:28,972 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/5760b309871049ee8b1e2b4d28cf0494 is 50, key is test_row_0/B:col10/1731310228265/Put/seqid=0 2024-11-11T07:30:28,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742110_1286 (size=12301) 2024-11-11T07:30:28,987 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9b6318133d324148abb4de9403a5b07d 2024-11-11T07:30:28,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742111_1287 (size=12949) 2024-11-11T07:30:29,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/905677c3e2e5480e89ed112bc43ca445 is 50, key is test_row_0/B:col10/1731310228635/Put/seqid=0 2024-11-11T07:30:29,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742112_1288 (size=12301) 2024-11-11T07:30:29,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310289018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310289020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310289020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310289024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,113 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/0bdecc321b2a469d92fcb960a68b2989 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/0bdecc321b2a469d92fcb960a68b2989 2024-11-11T07:30:29,120 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 385162ecea32f0e101e7709026aea910/C of 385162ecea32f0e101e7709026aea910 into 0bdecc321b2a469d92fcb960a68b2989(size=12.6 K), total size for store is 24.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:29,120 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:29,120 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/C, priority=12, startTime=1731310228051; duration=0sec 2024-11-11T07:30:29,121 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:29,121 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:C 2024-11-11T07:30:29,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310289135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,137 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310289135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310289144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,147 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,148 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310289144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T07:30:29,331 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49954 deadline: 1731310289327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,335 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_1' in 385162ecea32f0e101e7709026aea910 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8222 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,335 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310289345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310289345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,351 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310289354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310289354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,405 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/5760b309871049ee8b1e2b4d28cf0494 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5760b309871049ee8b1e2b4d28cf0494 2024-11-11T07:30:29,411 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/B of 385162ecea32f0e101e7709026aea910 into 5760b309871049ee8b1e2b4d28cf0494(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:29,411 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:29,411 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/B, priority=13, startTime=1731310228849; duration=0sec 2024-11-11T07:30:29,411 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:29,411 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:B 2024-11-11T07:30:29,428 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/905677c3e2e5480e89ed112bc43ca445 2024-11-11T07:30:29,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/b1f7e85221174bec8788c47d3c11746a is 50, key is test_row_0/C:col10/1731310228635/Put/seqid=0 2024-11-11T07:30:29,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742113_1289 (size=12301) 2024-11-11T07:30:29,524 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=311 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/b1f7e85221174bec8788c47d3c11746a 2024-11-11T07:30:29,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9b6318133d324148abb4de9403a5b07d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9b6318133d324148abb4de9403a5b07d 2024-11-11T07:30:29,535 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9b6318133d324148abb4de9403a5b07d, entries=150, sequenceid=311, filesize=12.0 K 2024-11-11T07:30:29,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/905677c3e2e5480e89ed112bc43ca445 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/905677c3e2e5480e89ed112bc43ca445 2024-11-11T07:30:29,542 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/905677c3e2e5480e89ed112bc43ca445, entries=150, sequenceid=311, filesize=12.0 K 2024-11-11T07:30:29,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/b1f7e85221174bec8788c47d3c11746a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/b1f7e85221174bec8788c47d3c11746a 2024-11-11T07:30:29,549 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/b1f7e85221174bec8788c47d3c11746a, entries=150, sequenceid=311, filesize=12.0 K 2024-11-11T07:30:29,550 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 385162ecea32f0e101e7709026aea910 in 612ms, sequenceid=311, compaction requested=true 2024-11-11T07:30:29,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:29,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:29,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-11T07:30:29,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-11T07:30:29,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-11T07:30:29,554 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 930 msec 2024-11-11T07:30:29,557 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 939 msec 2024-11-11T07:30:29,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T07:30:29,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:29,659 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:29,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:29,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:29,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:29,660 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:29,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:29,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9e0b4aa19b3241e4a6dd969205bcb60c is 50, key is test_row_0/A:col10/1731310229658/Put/seqid=0 2024-11-11T07:30:29,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310289684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,694 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310289687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,700 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310289691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,701 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310289693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742114_1290 (size=14741) 2024-11-11T07:30:29,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-11T07:30:29,755 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:29,759 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:29,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-11T07:30:29,761 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:29,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T07:30:29,761 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:29,761 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:29,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310289805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310289805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310289805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,813 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310289805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:29,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:29,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T07:30:29,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:30:29,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:29,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:29,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:29,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:29,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:29,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:30,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310290015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:30,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:30,026 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:30,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310290015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,026 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:30,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:30,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310290024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:30,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:30,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310290024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:30,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:30:30,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:30,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:30,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:30,073 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T07:30:30,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9e0b4aa19b3241e4a6dd969205bcb60c 2024-11-11T07:30:30,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/30c3492585ac4c058591d69769ff2da6 is 50, key is test_row_0/B:col10/1731310229658/Put/seqid=0 2024-11-11T07:30:30,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742115_1291 (size=12301) 2024-11-11T07:30:30,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/30c3492585ac4c058591d69769ff2da6 2024-11-11T07:30:30,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/936fbbcebba34deda1e68076dd6c28bb is 50, key is test_row_0/C:col10/1731310229658/Put/seqid=0 2024-11-11T07:30:30,225 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:30:30,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:30,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:30,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:30,226 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742116_1292 (size=12301) 2024-11-11T07:30:30,242 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=335 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/936fbbcebba34deda1e68076dd6c28bb 2024-11-11T07:30:30,249 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9e0b4aa19b3241e4a6dd969205bcb60c as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9e0b4aa19b3241e4a6dd969205bcb60c 2024-11-11T07:30:30,256 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9e0b4aa19b3241e4a6dd969205bcb60c, entries=200, sequenceid=335, filesize=14.4 K 2024-11-11T07:30:30,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/30c3492585ac4c058591d69769ff2da6 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/30c3492585ac4c058591d69769ff2da6 2024-11-11T07:30:30,263 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/30c3492585ac4c058591d69769ff2da6, entries=150, sequenceid=335, filesize=12.0 K 2024-11-11T07:30:30,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/936fbbcebba34deda1e68076dd6c28bb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/936fbbcebba34deda1e68076dd6c28bb 2024-11-11T07:30:30,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/936fbbcebba34deda1e68076dd6c28bb, entries=150, sequenceid=335, filesize=12.0 K 2024-11-11T07:30:30,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 385162ecea32f0e101e7709026aea910 in 609ms, sequenceid=335, compaction requested=true 2024-11-11T07:30:30,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:30,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:30,269 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:30,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:30,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:30,269 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:30,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:30,270 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:30,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:30,270 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:30,270 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/A is initiating minor compaction (all files) 2024-11-11T07:30:30,270 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/A in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:30,270 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/8a2ad4117fee46e98b46e5515ba11621, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9b6318133d324148abb4de9403a5b07d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9e0b4aa19b3241e4a6dd969205bcb60c] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=39.1 K 2024-11-11T07:30:30,270 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:30,270 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/B is initiating minor compaction (all files) 2024-11-11T07:30:30,270 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/B in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:30,270 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5760b309871049ee8b1e2b4d28cf0494, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/905677c3e2e5480e89ed112bc43ca445, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/30c3492585ac4c058591d69769ff2da6] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=36.7 K 2024-11-11T07:30:30,271 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a2ad4117fee46e98b46e5515ba11621, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731310228262 2024-11-11T07:30:30,271 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 5760b309871049ee8b1e2b4d28cf0494, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731310228262 2024-11-11T07:30:30,272 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b6318133d324148abb4de9403a5b07d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731310228632 2024-11-11T07:30:30,272 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 905677c3e2e5480e89ed112bc43ca445, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731310228632 2024-11-11T07:30:30,272 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9e0b4aa19b3241e4a6dd969205bcb60c, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731310229003 2024-11-11T07:30:30,273 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 30c3492585ac4c058591d69769ff2da6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731310229003 2024-11-11T07:30:30,297 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#A#compaction#251 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:30,297 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/1424d29836b54cd7b27a5a9308ec1fcb is 50, key is test_row_0/A:col10/1731310229658/Put/seqid=0 2024-11-11T07:30:30,300 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#B#compaction#252 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:30,301 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/045dea5b474a470ab658dcbe78b6ecc8 is 50, key is test_row_0/B:col10/1731310229658/Put/seqid=0 2024-11-11T07:30:30,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T07:30:30,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:30,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:30,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:30,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:30,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:30,339 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:30,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:30,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742117_1293 (size=13051) 2024-11-11T07:30:30,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742118_1294 (size=13051) 2024-11-11T07:30:30,367 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/1424d29836b54cd7b27a5a9308ec1fcb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1424d29836b54cd7b27a5a9308ec1fcb 2024-11-11T07:30:30,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/a630791b540d4ea7b4923f5547905cf5 is 50, key is test_row_0/A:col10/1731310230337/Put/seqid=0 2024-11-11T07:30:30,379 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/A of 385162ecea32f0e101e7709026aea910 into 1424d29836b54cd7b27a5a9308ec1fcb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:30,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:30:30,379 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:30,379 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/A, priority=13, startTime=1731310230269; duration=0sec 2024-11-11T07:30:30,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:30,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:30,379 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:30,379 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:A 2024-11-11T07:30:30,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:30,380 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:30,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,381 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:30,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,383 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/C is initiating minor compaction (all files) 2024-11-11T07:30:30,384 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/C in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:30,384 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/0bdecc321b2a469d92fcb960a68b2989, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/ddb417b8c43c49c48472142537247fa2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/b1f7e85221174bec8788c47d3c11746a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/936fbbcebba34deda1e68076dd6c28bb] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=48.6 K 2024-11-11T07:30:30,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T07:30:30,385 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bdecc321b2a469d92fcb960a68b2989, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=277, earliestPutTs=1731310227872 2024-11-11T07:30:30,386 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddb417b8c43c49c48472142537247fa2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731310228262 2024-11-11T07:30:30,386 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1f7e85221174bec8788c47d3c11746a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=311, earliestPutTs=1731310228632 2024-11-11T07:30:30,386 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 936fbbcebba34deda1e68076dd6c28bb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731310229003 2024-11-11T07:30:30,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742119_1295 (size=14741) 2024-11-11T07:30:30,403 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/a630791b540d4ea7b4923f5547905cf5 2024-11-11T07:30:30,403 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#C#compaction#254 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:30,404 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/baeff82d299f4054936c6506fa0a94e2 is 50, key is test_row_0/C:col10/1731310229658/Put/seqid=0 2024-11-11T07:30:30,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310230381, max timestamp=1731310230399Num uniques 859; 2024-11-11T07:30:30,405 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:30:30,416 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/de3c06c1df1446539fb4601909e8c614 is 50, key is test_row_0/B:col10/1731310230337/Put/seqid=0 2024-11-11T07:30:30,422 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:30,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310290410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310290412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,423 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:30,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:30,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310290418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:30,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,426 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:30,427 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:30,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310290420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:30,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742120_1296 (size=13017) 2024-11-11T07:30:30,444 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/baeff82d299f4054936c6506fa0a94e2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/baeff82d299f4054936c6506fa0a94e2 2024-11-11T07:30:30,451 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 385162ecea32f0e101e7709026aea910/C of 385162ecea32f0e101e7709026aea910 into baeff82d299f4054936c6506fa0a94e2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:30,451 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:30,451 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/C, priority=12, startTime=1731310230270; duration=0sec 2024-11-11T07:30:30,452 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:30,452 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:C 2024-11-11T07:30:30,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742121_1297 (size=12301) 2024-11-11T07:30:30,470 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/de3c06c1df1446539fb4601909e8c614 2024-11-11T07:30:30,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/6fbb67a8c5a1431fa7cb89617f2421df is 50, key is test_row_0/C:col10/1731310230337/Put/seqid=0 2024-11-11T07:30:30,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310290524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:30,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:30:30,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:30,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:30,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:30,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742122_1298 (size=12301) 2024-11-11T07:30:30,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,538 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310290535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:30,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,539 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:30,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310290535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:30,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:30,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310290535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:30,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:30,690 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:30:30,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:30,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:30,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:30,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:30,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310290734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:30,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:30,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310290745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:30,747 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:30,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310290745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:30,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310290745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:30,748 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:30,774 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/045dea5b474a470ab658dcbe78b6ecc8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/045dea5b474a470ab658dcbe78b6ecc8 2024-11-11T07:30:30,788 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/B of 385162ecea32f0e101e7709026aea910 into 045dea5b474a470ab658dcbe78b6ecc8(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:30,788 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:30,788 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/B, priority=13, startTime=1731310230269; duration=0sec 2024-11-11T07:30:30,788 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:30,788 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:B 2024-11-11T07:30:30,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:30:30,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:30,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:30,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:30,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:30,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T07:30:30,939 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=347 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/6fbb67a8c5a1431fa7cb89617f2421df 2024-11-11T07:30:30,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/a630791b540d4ea7b4923f5547905cf5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/a630791b540d4ea7b4923f5547905cf5 2024-11-11T07:30:30,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/a630791b540d4ea7b4923f5547905cf5, entries=200, sequenceid=347, filesize=14.4 K 2024-11-11T07:30:30,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/de3c06c1df1446539fb4601909e8c614 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/de3c06c1df1446539fb4601909e8c614 2024-11-11T07:30:30,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/de3c06c1df1446539fb4601909e8c614, entries=150, sequenceid=347, filesize=12.0 K 2024-11-11T07:30:30,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/6fbb67a8c5a1431fa7cb89617f2421df as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/6fbb67a8c5a1431fa7cb89617f2421df 2024-11-11T07:30:30,965 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/6fbb67a8c5a1431fa7cb89617f2421df, entries=150, sequenceid=347, filesize=12.0 K 2024-11-11T07:30:30,966 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=116.29 KB/119080 for 385162ecea32f0e101e7709026aea910 in 627ms, sequenceid=347, compaction requested=false 2024-11-11T07:30:30,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:30,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-11T07:30:30,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:30,996 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=116.29 KB heapSize=341.80 KB 2024-11-11T07:30:30,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:30,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:30,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:30,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:30,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:30,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:31,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/1bc3ed91174e4b7c8cfa4c34262e2dde is 50, key is test_row_0/A:col10/1731310230414/Put/seqid=0 2024-11-11T07:30:31,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742123_1299 (size=12301) 2024-11-11T07:30:31,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:31,051 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/1bc3ed91174e4b7c8cfa4c34262e2dde 2024-11-11T07:30:31,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:31,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/c94594db10c747019ffb157e95d219a3 is 50, key is test_row_0/B:col10/1731310230414/Put/seqid=0 2024-11-11T07:30:31,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310291078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310291080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,093 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,094 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,095 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310291084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,096 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310291093, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,101 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742124_1300 (size=12301) 2024-11-11T07:30:31,103 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/c94594db10c747019ffb157e95d219a3 2024-11-11T07:30:31,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/cbc41f68ee9547e99235df533c5b661e is 50, key is test_row_0/C:col10/1731310230414/Put/seqid=0 2024-11-11T07:30:31,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742125_1301 (size=12301) 2024-11-11T07:30:31,163 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/cbc41f68ee9547e99235df533c5b661e 2024-11-11T07:30:31,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/1bc3ed91174e4b7c8cfa4c34262e2dde as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1bc3ed91174e4b7c8cfa4c34262e2dde 2024-11-11T07:30:31,173 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1bc3ed91174e4b7c8cfa4c34262e2dde, entries=150, sequenceid=375, filesize=12.0 K 2024-11-11T07:30:31,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/c94594db10c747019ffb157e95d219a3 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/c94594db10c747019ffb157e95d219a3 2024-11-11T07:30:31,179 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/c94594db10c747019ffb157e95d219a3, entries=150, sequenceid=375, filesize=12.0 K 2024-11-11T07:30:31,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/cbc41f68ee9547e99235df533c5b661e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/cbc41f68ee9547e99235df533c5b661e 2024-11-11T07:30:31,187 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/cbc41f68ee9547e99235df533c5b661e, entries=150, sequenceid=375, filesize=12.0 K 2024-11-11T07:30:31,188 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~116.29 KB/119080, heapSize ~341.75 KB/349952, currentSize=60.38 KB/61830 for 385162ecea32f0e101e7709026aea910 in 192ms, sequenceid=375, compaction requested=true 2024-11-11T07:30:31,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:31,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:31,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-11T07:30:31,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-11T07:30:31,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-11T07:30:31,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4290 sec 2024-11-11T07:30:31,200 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 1.4380 sec 2024-11-11T07:30:31,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:31,208 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T07:30:31,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:31,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:31,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:31,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:31,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:31,208 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:31,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/5c3890646b62463796fbb7bb39c35f28 is 50, key is test_row_0/A:col10/1731310231062/Put/seqid=0 2024-11-11T07:30:31,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742126_1302 (size=14741) 2024-11-11T07:30:31,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/5c3890646b62463796fbb7bb39c35f28 2024-11-11T07:30:31,252 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/11755c2066e4438684163cf74c0dcfc8 is 50, key is test_row_0/B:col10/1731310231062/Put/seqid=0 2024-11-11T07:30:31,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310291254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310291254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,265 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,265 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310291264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310291265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,272 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742127_1303 (size=12301) 2024-11-11T07:30:31,279 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/11755c2066e4438684163cf74c0dcfc8 2024-11-11T07:30:31,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/38294795e74843a2a1c84e41688491e3 is 50, key is test_row_0/C:col10/1731310231062/Put/seqid=0 2024-11-11T07:30:31,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742128_1304 (size=12301) 2024-11-11T07:30:31,316 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/38294795e74843a2a1c84e41688491e3 2024-11-11T07:30:31,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/5c3890646b62463796fbb7bb39c35f28 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/5c3890646b62463796fbb7bb39c35f28 2024-11-11T07:30:31,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/5c3890646b62463796fbb7bb39c35f28, entries=200, sequenceid=388, filesize=14.4 K 2024-11-11T07:30:31,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/11755c2066e4438684163cf74c0dcfc8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/11755c2066e4438684163cf74c0dcfc8 2024-11-11T07:30:31,353 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/11755c2066e4438684163cf74c0dcfc8, entries=150, sequenceid=388, filesize=12.0 K 2024-11-11T07:30:31,354 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/38294795e74843a2a1c84e41688491e3 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/38294795e74843a2a1c84e41688491e3 2024-11-11T07:30:31,359 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/38294795e74843a2a1c84e41688491e3, entries=150, sequenceid=388, filesize=12.0 K 2024-11-11T07:30:31,361 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 385162ecea32f0e101e7709026aea910 in 153ms, sequenceid=388, compaction requested=true 2024-11-11T07:30:31,361 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:31,361 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:31,363 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:31,363 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54834 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:31,363 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/A is initiating minor compaction (all files) 2024-11-11T07:30:31,363 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/A in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:31,363 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1424d29836b54cd7b27a5a9308ec1fcb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/a630791b540d4ea7b4923f5547905cf5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1bc3ed91174e4b7c8cfa4c34262e2dde, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/5c3890646b62463796fbb7bb39c35f28] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=53.5 K 2024-11-11T07:30:31,365 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1424d29836b54cd7b27a5a9308ec1fcb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731310229003 2024-11-11T07:30:31,366 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting a630791b540d4ea7b4923f5547905cf5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731310229682 2024-11-11T07:30:31,366 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1bc3ed91174e4b7c8cfa4c34262e2dde, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731310230410 2024-11-11T07:30:31,367 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:31,367 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:31,368 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c3890646b62463796fbb7bb39c35f28, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731310231062 2024-11-11T07:30:31,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:31,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:31,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:31,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:31,369 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:31,369 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/B is initiating minor compaction (all files) 2024-11-11T07:30:31,369 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/B in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:31,369 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/045dea5b474a470ab658dcbe78b6ecc8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/de3c06c1df1446539fb4601909e8c614, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/c94594db10c747019ffb157e95d219a3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/11755c2066e4438684163cf74c0dcfc8] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=48.8 K 2024-11-11T07:30:31,370 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 045dea5b474a470ab658dcbe78b6ecc8, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731310229003 2024-11-11T07:30:31,372 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting de3c06c1df1446539fb4601909e8c614, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731310229682 2024-11-11T07:30:31,372 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting c94594db10c747019ffb157e95d219a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731310230410 2024-11-11T07:30:31,373 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 11755c2066e4438684163cf74c0dcfc8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731310231062 2024-11-11T07:30:31,383 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#A#compaction#263 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:31,384 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/1353829d62c94225ad887908d5673936 is 50, key is test_row_0/A:col10/1731310231062/Put/seqid=0 2024-11-11T07:30:31,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:31,387 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:30:31,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:31,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:31,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:31,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:31,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:31,388 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:31,394 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#B#compaction#264 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:31,394 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/f34c9d54a7aa44c8acbced2029a8a07f is 50, key is test_row_0/B:col10/1731310231062/Put/seqid=0 2024-11-11T07:30:31,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/4487442c570849e89ef341342568a978 is 50, key is test_row_0/A:col10/1731310231385/Put/seqid=0 2024-11-11T07:30:31,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742129_1305 (size=13187) 2024-11-11T07:30:31,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742130_1306 (size=13187) 2024-11-11T07:30:31,438 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/1353829d62c94225ad887908d5673936 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1353829d62c94225ad887908d5673936 2024-11-11T07:30:31,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310291429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,447 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/f34c9d54a7aa44c8acbced2029a8a07f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f34c9d54a7aa44c8acbced2029a8a07f 2024-11-11T07:30:31,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310291442, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742131_1307 (size=12301) 2024-11-11T07:30:31,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310291443, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,461 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/4487442c570849e89ef341342568a978 2024-11-11T07:30:31,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310291444, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,461 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,472 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 385162ecea32f0e101e7709026aea910/B of 385162ecea32f0e101e7709026aea910 into f34c9d54a7aa44c8acbced2029a8a07f(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:31,472 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 385162ecea32f0e101e7709026aea910/A of 385162ecea32f0e101e7709026aea910 into 1353829d62c94225ad887908d5673936(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:31,472 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:31,472 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:31,472 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/B, priority=12, startTime=1731310231367; duration=0sec 2024-11-11T07:30:31,472 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/A, priority=12, startTime=1731310231361; duration=0sec 2024-11-11T07:30:31,472 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:31,472 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:A 2024-11-11T07:30:31,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,472 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:31,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,473 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:31,473 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:B 2024-11-11T07:30:31,474 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:31,474 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/C is initiating minor compaction (all files) 2024-11-11T07:30:31,474 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/C in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:31,474 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/baeff82d299f4054936c6506fa0a94e2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/6fbb67a8c5a1431fa7cb89617f2421df, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/cbc41f68ee9547e99235df533c5b661e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/38294795e74843a2a1c84e41688491e3] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=48.8 K 2024-11-11T07:30:31,474 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting baeff82d299f4054936c6506fa0a94e2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=335, earliestPutTs=1731310229003 2024-11-11T07:30:31,475 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fbb67a8c5a1431fa7cb89617f2421df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=347, earliestPutTs=1731310229682 2024-11-11T07:30:31,476 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting cbc41f68ee9547e99235df533c5b661e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731310230410 2024-11-11T07:30:31,476 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 38294795e74843a2a1c84e41688491e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731310231062 2024-11-11T07:30:31,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/b8f6f966367c41daa7414a8747df1ccb is 50, key is test_row_0/B:col10/1731310231385/Put/seqid=0 2024-11-11T07:30:31,489 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#C#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:31,490 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/6d21ae080f894e0287674a0f5c54c416 is 50, key is test_row_0/C:col10/1731310231062/Put/seqid=0 2024-11-11T07:30:31,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742132_1308 (size=12301) 2024-11-11T07:30:31,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/b8f6f966367c41daa7414a8747df1ccb 2024-11-11T07:30:31,525 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/5c1e72f873f1423093bb1cfb830e96c7 is 50, key is test_row_0/C:col10/1731310231385/Put/seqid=0 2024-11-11T07:30:31,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742133_1309 (size=13153) 2024-11-11T07:30:31,532 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/6d21ae080f894e0287674a0f5c54c416 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/6d21ae080f894e0287674a0f5c54c416 2024-11-11T07:30:31,540 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 385162ecea32f0e101e7709026aea910/C of 385162ecea32f0e101e7709026aea910 into 6d21ae080f894e0287674a0f5c54c416(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:31,540 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:31,540 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/C, priority=12, startTime=1731310231368; duration=0sec 2024-11-11T07:30:31,540 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:31,540 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:C 2024-11-11T07:30:31,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742134_1310 (size=12301) 2024-11-11T07:30:31,556 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=408 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/5c1e72f873f1423093bb1cfb830e96c7 2024-11-11T07:30:31,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310291555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,557 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/4487442c570849e89ef341342568a978 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/4487442c570849e89ef341342568a978 2024-11-11T07:30:31,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/4487442c570849e89ef341342568a978, entries=150, sequenceid=408, filesize=12.0 K 2024-11-11T07:30:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310291565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,571 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,571 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310291565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,571 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,572 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/b8f6f966367c41daa7414a8747df1ccb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b8f6f966367c41daa7414a8747df1ccb 2024-11-11T07:30:31,576 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b8f6f966367c41daa7414a8747df1ccb, entries=150, sequenceid=408, filesize=12.0 K 2024-11-11T07:30:31,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/5c1e72f873f1423093bb1cfb830e96c7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/5c1e72f873f1423093bb1cfb830e96c7 2024-11-11T07:30:31,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/5c1e72f873f1423093bb1cfb830e96c7, entries=150, sequenceid=408, filesize=12.0 K 2024-11-11T07:30:31,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 385162ecea32f0e101e7709026aea910 in 196ms, sequenceid=408, compaction requested=false 2024-11-11T07:30:31,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:31,587 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T07:30:31,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:31,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:31,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:31,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:31,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:31,587 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:31,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/b9afd3b141af47fa8aff0c79e815b1b4 is 50, key is test_row_0/A:col10/1731310231585/Put/seqid=0 2024-11-11T07:30:31,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742135_1311 (size=14741) 2024-11-11T07:30:31,631 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/b9afd3b141af47fa8aff0c79e815b1b4 2024-11-11T07:30:31,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/f4a63c4f2db0495db3598dcfab130911 is 50, key is test_row_0/B:col10/1731310231585/Put/seqid=0 2024-11-11T07:30:31,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742136_1312 (size=12301) 2024-11-11T07:30:31,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/f4a63c4f2db0495db3598dcfab130911 2024-11-11T07:30:31,738 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/ae83d5ca94ef43c9817c97d79605eb7e is 50, key is test_row_0/C:col10/1731310231585/Put/seqid=0 2024-11-11T07:30:31,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310291744, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,751 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310291765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,774 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,776 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310291775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,777 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310291775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,778 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742137_1313 (size=12301) 2024-11-11T07:30:31,857 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:31,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310291854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:31,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:31,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:31,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-11T07:30:31,907 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:31,907 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:31,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-11T07:30:31,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T07:30:31,914 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:31,915 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:31,915 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:32,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T07:30:32,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:32,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310292065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:32,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:30:32,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:32,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:32,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:32,071 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:32,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:32,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:32,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:32,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310292084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:32,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:32,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,088 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:32,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310292085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:32,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:32,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:32,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310292085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,090 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:32,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=424 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/ae83d5ca94ef43c9817c97d79605eb7e 2024-11-11T07:30:32,189 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/b9afd3b141af47fa8aff0c79e815b1b4 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/b9afd3b141af47fa8aff0c79e815b1b4 2024-11-11T07:30:32,194 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/b9afd3b141af47fa8aff0c79e815b1b4, entries=200, sequenceid=424, filesize=14.4 K 2024-11-11T07:30:32,196 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/f4a63c4f2db0495db3598dcfab130911 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f4a63c4f2db0495db3598dcfab130911 2024-11-11T07:30:32,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f4a63c4f2db0495db3598dcfab130911, entries=150, sequenceid=424, filesize=12.0 K 2024-11-11T07:30:32,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/ae83d5ca94ef43c9817c97d79605eb7e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/ae83d5ca94ef43c9817c97d79605eb7e 2024-11-11T07:30:32,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/ae83d5ca94ef43c9817c97d79605eb7e, entries=150, sequenceid=424, filesize=12.0 K 2024-11-11T07:30:32,210 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 385162ecea32f0e101e7709026aea910 in 623ms, sequenceid=424, compaction requested=true 2024-11-11T07:30:32,211 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:32,211 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:32,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:32,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:32,211 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:32,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:32,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:32,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:32,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:32,212 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:32,212 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/A is initiating minor compaction (all files) 2024-11-11T07:30:32,212 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/A in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:32,212 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1353829d62c94225ad887908d5673936, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/4487442c570849e89ef341342568a978, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/b9afd3b141af47fa8aff0c79e815b1b4] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=39.3 K 2024-11-11T07:30:32,213 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:32,213 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/B is initiating minor compaction (all files) 2024-11-11T07:30:32,213 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/B in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:32,213 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f34c9d54a7aa44c8acbced2029a8a07f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b8f6f966367c41daa7414a8747df1ccb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f4a63c4f2db0495db3598dcfab130911] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=36.9 K 2024-11-11T07:30:32,213 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1353829d62c94225ad887908d5673936, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731310231062 2024-11-11T07:30:32,213 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f34c9d54a7aa44c8acbced2029a8a07f, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731310231062 2024-11-11T07:30:32,214 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4487442c570849e89ef341342568a978, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731310231249 2024-11-11T07:30:32,214 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting b8f6f966367c41daa7414a8747df1ccb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731310231249 2024-11-11T07:30:32,215 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting b9afd3b141af47fa8aff0c79e815b1b4, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731310231419 2024-11-11T07:30:32,215 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f4a63c4f2db0495db3598dcfab130911, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731310231429 2024-11-11T07:30:32,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T07:30:32,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-11T07:30:32,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:32,235 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T07:30:32,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:32,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:32,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:32,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:32,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:32,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:32,247 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#A#compaction#272 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:32,247 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/03607a5b1d7a44898a5567d76f8fd15d is 50, key is test_row_0/A:col10/1731310231585/Put/seqid=0 2024-11-11T07:30:32,253 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#B#compaction#273 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:32,253 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/67db11d5662948f5a20eceb30068a6dd is 50, key is test_row_0/B:col10/1731310231585/Put/seqid=0 2024-11-11T07:30:32,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9966664370da4781bb2989059e9f4ec7 is 50, key is test_row_0/A:col10/1731310231742/Put/seqid=0 2024-11-11T07:30:32,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742138_1314 (size=13289) 2024-11-11T07:30:32,316 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/03607a5b1d7a44898a5567d76f8fd15d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/03607a5b1d7a44898a5567d76f8fd15d 2024-11-11T07:30:32,322 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/A of 385162ecea32f0e101e7709026aea910 into 03607a5b1d7a44898a5567d76f8fd15d(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:32,322 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:32,322 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/A, priority=13, startTime=1731310232211; duration=0sec 2024-11-11T07:30:32,322 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:32,322 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:A 2024-11-11T07:30:32,322 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:32,323 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:32,323 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/C is initiating minor compaction (all files) 2024-11-11T07:30:32,323 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/C in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:32,323 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/6d21ae080f894e0287674a0f5c54c416, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/5c1e72f873f1423093bb1cfb830e96c7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/ae83d5ca94ef43c9817c97d79605eb7e] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=36.9 K 2024-11-11T07:30:32,324 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d21ae080f894e0287674a0f5c54c416, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731310231062 2024-11-11T07:30:32,324 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c1e72f873f1423093bb1cfb830e96c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=408, earliestPutTs=1731310231249 2024-11-11T07:30:32,324 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae83d5ca94ef43c9817c97d79605eb7e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731310231429 2024-11-11T07:30:32,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742139_1315 (size=13289) 2024-11-11T07:30:32,336 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/67db11d5662948f5a20eceb30068a6dd as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/67db11d5662948f5a20eceb30068a6dd 2024-11-11T07:30:32,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742140_1316 (size=12301) 2024-11-11T07:30:32,342 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9966664370da4781bb2989059e9f4ec7 2024-11-11T07:30:32,343 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/B of 385162ecea32f0e101e7709026aea910 into 67db11d5662948f5a20eceb30068a6dd(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:32,343 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:32,343 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/B, priority=13, startTime=1731310232211; duration=0sec 2024-11-11T07:30:32,343 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:32,343 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:B 2024-11-11T07:30:32,357 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#C#compaction#275 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:32,357 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/5a036da7567f40b2b1390397bb1de436 is 50, key is test_row_0/C:col10/1731310231585/Put/seqid=0 2024-11-11T07:30:32,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/cc4a90225b2d42d6a86f4c8830745d9f is 50, key is test_row_0/B:col10/1731310231742/Put/seqid=0 2024-11-11T07:30:32,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:32,378 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:32,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742141_1317 (size=13255) 2024-11-11T07:30:32,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742142_1318 (size=12301) 2024-11-11T07:30:32,416 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/5a036da7567f40b2b1390397bb1de436 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/5a036da7567f40b2b1390397bb1de436 2024-11-11T07:30:32,420 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/cc4a90225b2d42d6a86f4c8830745d9f 2024-11-11T07:30:32,431 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/C of 385162ecea32f0e101e7709026aea910 into 5a036da7567f40b2b1390397bb1de436(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:32,431 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:32,431 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/C, priority=13, startTime=1731310232211; duration=0sec 2024-11-11T07:30:32,431 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:32,431 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:C 2024-11-11T07:30:32,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/279878a602da442b8295b66e50f29279 is 50, key is test_row_0/C:col10/1731310231742/Put/seqid=0 2024-11-11T07:30:32,459 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:32,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310292449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:32,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:32,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742143_1319 (size=12301) 2024-11-11T07:30:32,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T07:30:32,567 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310292565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:32,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310292595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:32,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310292595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,599 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,600 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:32,605 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:32,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310292595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,611 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:32,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:32,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310292775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:32,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:32,932 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=443 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/279878a602da442b8295b66e50f29279 2024-11-11T07:30:32,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/9966664370da4781bb2989059e9f4ec7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9966664370da4781bb2989059e9f4ec7 2024-11-11T07:30:32,944 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9966664370da4781bb2989059e9f4ec7, entries=150, sequenceid=443, filesize=12.0 K 2024-11-11T07:30:32,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/cc4a90225b2d42d6a86f4c8830745d9f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/cc4a90225b2d42d6a86f4c8830745d9f 2024-11-11T07:30:32,969 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/cc4a90225b2d42d6a86f4c8830745d9f, entries=150, sequenceid=443, filesize=12.0 K 2024-11-11T07:30:32,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/279878a602da442b8295b66e50f29279 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/279878a602da442b8295b66e50f29279 2024-11-11T07:30:32,975 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/279878a602da442b8295b66e50f29279, entries=150, sequenceid=443, filesize=12.0 K 2024-11-11T07:30:32,977 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 385162ecea32f0e101e7709026aea910 in 743ms, sequenceid=443, compaction requested=false 2024-11-11T07:30:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:32,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-11T07:30:32,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-11T07:30:32,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-11T07:30:32,982 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0630 sec 2024-11-11T07:30:32,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.0770 sec 2024-11-11T07:30:33,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-11T07:30:33,045 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:33,045 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:33,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-11T07:30:33,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T07:30:33,048 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:33,048 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:33,049 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:33,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T07:30:33,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:33,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:33,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:33,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:33,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:33,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:33,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:33,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/138b56b6e446497083996c70c8817011 is 50, key is test_row_0/A:col10/1731310232444/Put/seqid=0 2024-11-11T07:30:33,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742144_1320 (size=14741) 2024-11-11T07:30:33,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T07:30:33,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:33,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310293185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:33,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:33,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:33,192 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:33,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T07:30:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:33,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:33,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:33,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:33,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:33,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310293295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:33,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:33,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:33,299 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:33,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T07:30:33,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:33,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:33,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:33,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:33,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:33,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:33,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T07:30:33,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:33,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310293505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:33,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:33,510 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T07:30:33,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:33,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:33,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:33,510 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:33,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:33,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:33,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/138b56b6e446497083996c70c8817011 2024-11-11T07:30:33,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/8261837467604448a57e0a2945c9163b is 50, key is test_row_0/B:col10/1731310232444/Put/seqid=0 2024-11-11T07:30:33,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742145_1321 (size=12301) 2024-11-11T07:30:33,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/8261837467604448a57e0a2945c9163b 2024-11-11T07:30:33,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/2b0cb89191b44b1bad5dbcad6cc69c0d is 50, key is test_row_0/C:col10/1731310232444/Put/seqid=0 2024-11-11T07:30:33,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:33,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310293614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:33,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:33,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310293615, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:33,619 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_0' in 385162ecea32f0e101e7709026aea910 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:33,620 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_0' in 385162ecea32f0e101e7709026aea910 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:33,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:33,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:33,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310293625, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:33,628 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_0' in 385162ecea32f0e101e7709026aea910 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:33,629 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:33,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742146_1322 (size=12301) 2024-11-11T07:30:33,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=460 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/2b0cb89191b44b1bad5dbcad6cc69c0d 2024-11-11T07:30:33,663 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T07:30:33,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:33,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:33,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:33,664 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:33,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:33,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:33,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/138b56b6e446497083996c70c8817011 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/138b56b6e446497083996c70c8817011 2024-11-11T07:30:33,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T07:30:33,680 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/138b56b6e446497083996c70c8817011, entries=200, sequenceid=460, filesize=14.4 K 2024-11-11T07:30:33,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/8261837467604448a57e0a2945c9163b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/8261837467604448a57e0a2945c9163b 2024-11-11T07:30:33,687 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/8261837467604448a57e0a2945c9163b, entries=150, sequenceid=460, filesize=12.0 K 2024-11-11T07:30:33,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/2b0cb89191b44b1bad5dbcad6cc69c0d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/2b0cb89191b44b1bad5dbcad6cc69c0d 2024-11-11T07:30:33,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/2b0cb89191b44b1bad5dbcad6cc69c0d, entries=150, sequenceid=460, filesize=12.0 K 2024-11-11T07:30:33,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 385162ecea32f0e101e7709026aea910 in 610ms, sequenceid=460, compaction requested=true 2024-11-11T07:30:33,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:33,697 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:33,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:33,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:33,698 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:33,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:33,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:33,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:33,698 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:33,698 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:33,698 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:33,698 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/A is initiating minor compaction (all files) 2024-11-11T07:30:33,699 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/B is initiating minor compaction (all files) 2024-11-11T07:30:33,699 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/A in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:33,699 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/B in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:33,699 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/67db11d5662948f5a20eceb30068a6dd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/cc4a90225b2d42d6a86f4c8830745d9f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/8261837467604448a57e0a2945c9163b] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=37.0 K 2024-11-11T07:30:33,699 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/03607a5b1d7a44898a5567d76f8fd15d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9966664370da4781bb2989059e9f4ec7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/138b56b6e446497083996c70c8817011] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=39.4 K 2024-11-11T07:30:33,699 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 67db11d5662948f5a20eceb30068a6dd, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731310231429 2024-11-11T07:30:33,699 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 03607a5b1d7a44898a5567d76f8fd15d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731310231429 2024-11-11T07:30:33,700 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 9966664370da4781bb2989059e9f4ec7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731310231681 2024-11-11T07:30:33,700 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc4a90225b2d42d6a86f4c8830745d9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731310231681 2024-11-11T07:30:33,701 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 138b56b6e446497083996c70c8817011, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731310232412 2024-11-11T07:30:33,701 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8261837467604448a57e0a2945c9163b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731310232439 2024-11-11T07:30:33,715 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#B#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:33,715 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/536b0927b8b54f7ebb6b6ab60c8d35d5 is 50, key is test_row_0/B:col10/1731310232444/Put/seqid=0 2024-11-11T07:30:33,718 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#A#compaction#282 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:33,719 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/cfd9757a89d248d49f859a330a7d76e2 is 50, key is test_row_0/A:col10/1731310232444/Put/seqid=0 2024-11-11T07:30:33,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742147_1323 (size=13391) 2024-11-11T07:30:33,759 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/536b0927b8b54f7ebb6b6ab60c8d35d5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/536b0927b8b54f7ebb6b6ab60c8d35d5 2024-11-11T07:30:33,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742148_1324 (size=13391) 2024-11-11T07:30:33,766 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/B of 385162ecea32f0e101e7709026aea910 into 536b0927b8b54f7ebb6b6ab60c8d35d5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:33,766 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:33,766 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/B, priority=13, startTime=1731310233697; duration=0sec 2024-11-11T07:30:33,766 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:33,766 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:B 2024-11-11T07:30:33,774 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/cfd9757a89d248d49f859a330a7d76e2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/cfd9757a89d248d49f859a330a7d76e2 2024-11-11T07:30:33,774 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:33,776 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:33,776 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/C is initiating minor compaction (all files) 2024-11-11T07:30:33,776 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/C in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:33,776 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/5a036da7567f40b2b1390397bb1de436, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/279878a602da442b8295b66e50f29279, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/2b0cb89191b44b1bad5dbcad6cc69c0d] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=37.0 K 2024-11-11T07:30:33,776 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a036da7567f40b2b1390397bb1de436, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=424, earliestPutTs=1731310231429 2024-11-11T07:30:33,777 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 279878a602da442b8295b66e50f29279, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=443, earliestPutTs=1731310231681 2024-11-11T07:30:33,777 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b0cb89191b44b1bad5dbcad6cc69c0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731310232439 2024-11-11T07:30:33,779 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/A of 385162ecea32f0e101e7709026aea910 into cfd9757a89d248d49f859a330a7d76e2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:33,779 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:33,779 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/A, priority=13, startTime=1731310233697; duration=0sec 2024-11-11T07:30:33,779 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:33,779 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:A 2024-11-11T07:30:33,807 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#C#compaction#283 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:33,807 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/a903f6db85e34e8781fb8eac35ec7e80 is 50, key is test_row_0/C:col10/1731310232444/Put/seqid=0 2024-11-11T07:30:33,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-11T07:30:33,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:33,817 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T07:30:33,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:33,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:33,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:33,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:33,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:33,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:33,822 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:33,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:33,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/39a2f59501704755832540c23f702213 is 50, key is test_row_0/A:col10/1731310233163/Put/seqid=0 2024-11-11T07:30:33,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742149_1325 (size=13357) 2024-11-11T07:30:33,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742150_1326 (size=12301) 2024-11-11T07:30:33,900 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/39a2f59501704755832540c23f702213 2024-11-11T07:30:33,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/5aab4aeb8ab7405eadfcbcbbe6b47f68 is 50, key is test_row_0/B:col10/1731310233163/Put/seqid=0 2024-11-11T07:30:33,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:33,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310293910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:33,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:33,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742151_1327 (size=12301) 2024-11-11T07:30:33,918 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/5aab4aeb8ab7405eadfcbcbbe6b47f68 2024-11-11T07:30:33,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/a2c3b883c4f341beb410272e3375417b is 50, key is test_row_0/C:col10/1731310233163/Put/seqid=0 2024-11-11T07:30:33,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742152_1328 (size=12301) 2024-11-11T07:30:33,993 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=480 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/a2c3b883c4f341beb410272e3375417b 2024-11-11T07:30:33,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/39a2f59501704755832540c23f702213 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/39a2f59501704755832540c23f702213 2024-11-11T07:30:34,005 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/39a2f59501704755832540c23f702213, entries=150, sequenceid=480, filesize=12.0 K 2024-11-11T07:30:34,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/5aab4aeb8ab7405eadfcbcbbe6b47f68 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5aab4aeb8ab7405eadfcbcbbe6b47f68 2024-11-11T07:30:34,010 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5aab4aeb8ab7405eadfcbcbbe6b47f68, entries=150, sequenceid=480, filesize=12.0 K 2024-11-11T07:30:34,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/a2c3b883c4f341beb410272e3375417b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a2c3b883c4f341beb410272e3375417b 2024-11-11T07:30:34,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310294024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:34,031 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:34,035 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a2c3b883c4f341beb410272e3375417b, entries=150, sequenceid=480, filesize=12.0 K 2024-11-11T07:30:34,036 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 385162ecea32f0e101e7709026aea910 in 218ms, sequenceid=480, compaction requested=false 2024-11-11T07:30:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:34,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-11T07:30:34,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-11T07:30:34,045 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-11T07:30:34,045 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 995 msec 2024-11-11T07:30:34,046 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0000 sec 2024-11-11T07:30:34,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-11T07:30:34,190 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:34,191 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:34,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-11T07:30:34,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T07:30:34,202 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:34,206 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:34,207 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:34,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:34,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T07:30:34,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:34,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:34,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:34,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:34,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:34,251 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:34,268 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/a903f6db85e34e8781fb8eac35ec7e80 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a903f6db85e34e8781fb8eac35ec7e80 2024-11-11T07:30:34,271 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/1444c34086274c18b7425ee010d3714e is 50, key is test_row_0/A:col10/1731310233897/Put/seqid=0 2024-11-11T07:30:34,282 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/C of 385162ecea32f0e101e7709026aea910 into a903f6db85e34e8781fb8eac35ec7e80(size=13.0 K), total size for store is 25.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:34,282 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:34,282 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/C, priority=13, startTime=1731310233698; duration=0sec 2024-11-11T07:30:34,283 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:34,283 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:C 2024-11-11T07:30:34,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742153_1329 (size=12301) 2024-11-11T07:30:34,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T07:30:34,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:34,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 208 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310294354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:34,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:34,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T07:30:34,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:34,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:34,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:34,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:34,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:34,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:34,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:34,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310294465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:34,471 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:34,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:34,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T07:30:34,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T07:30:34,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:34,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:34,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:34,531 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:34,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:34,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:34,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310294675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:34,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:34,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:34,678 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:34,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T07:30:34,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:34,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:34,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:34,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:34,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:34,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:34,689 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/1444c34086274c18b7425ee010d3714e 2024-11-11T07:30:34,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/a1f9d5acba87405889d20aca2c2f10f0 is 50, key is test_row_0/B:col10/1731310233897/Put/seqid=0 2024-11-11T07:30:34,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742154_1330 (size=12301) 2024-11-11T07:30:34,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T07:30:34,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T07:30:34,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:34,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:34,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:34,839 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:34,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:34,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:34,994 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:34,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310294987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:34,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:34,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:34,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:34,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T07:30:34,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:34,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:34,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:34,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:34,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:34,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:35,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/a1f9d5acba87405889d20aca2c2f10f0 2024-11-11T07:30:35,152 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/854447b9541c41daa264df39c61a732d is 50, key is test_row_0/C:col10/1731310233897/Put/seqid=0 2024-11-11T07:30:35,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T07:30:35,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:35,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:35,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:35,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:35,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:35,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:35,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742155_1331 (size=12301) 2024-11-11T07:30:35,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T07:30:35,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:35,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:35,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:35,322 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:35,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T07:30:35,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T07:30:35,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:35,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:35,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:35,475 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:35,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:35,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:35,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:35,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310295504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:35,509 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:35,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=496 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/854447b9541c41daa264df39c61a732d 2024-11-11T07:30:35,597 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/1444c34086274c18b7425ee010d3714e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1444c34086274c18b7425ee010d3714e 2024-11-11T07:30:35,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1444c34086274c18b7425ee010d3714e, entries=150, sequenceid=496, filesize=12.0 K 2024-11-11T07:30:35,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/a1f9d5acba87405889d20aca2c2f10f0 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/a1f9d5acba87405889d20aca2c2f10f0 2024-11-11T07:30:35,606 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/a1f9d5acba87405889d20aca2c2f10f0, entries=150, sequenceid=496, filesize=12.0 K 2024-11-11T07:30:35,607 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/854447b9541c41daa264df39c61a732d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/854447b9541c41daa264df39c61a732d 2024-11-11T07:30:35,613 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/854447b9541c41daa264df39c61a732d, entries=150, sequenceid=496, filesize=12.0 K 2024-11-11T07:30:35,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 385162ecea32f0e101e7709026aea910 in 1372ms, sequenceid=496, compaction requested=true 2024-11-11T07:30:35,614 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:35,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:35,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:35,614 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:35,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:35,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:35,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:35,614 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:35,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:35,617 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:35,617 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/B is initiating minor compaction (all files) 2024-11-11T07:30:35,617 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/B in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:35,617 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/536b0927b8b54f7ebb6b6ab60c8d35d5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5aab4aeb8ab7405eadfcbcbbe6b47f68, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/a1f9d5acba87405889d20aca2c2f10f0] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=37.1 K 2024-11-11T07:30:35,618 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:35,618 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/A is initiating minor compaction (all files) 2024-11-11T07:30:35,618 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/A in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:35,618 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/cfd9757a89d248d49f859a330a7d76e2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/39a2f59501704755832540c23f702213, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1444c34086274c18b7425ee010d3714e] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=37.1 K 2024-11-11T07:30:35,618 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting cfd9757a89d248d49f859a330a7d76e2, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731310232439 2024-11-11T07:30:35,619 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 536b0927b8b54f7ebb6b6ab60c8d35d5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731310232439 2024-11-11T07:30:35,619 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39a2f59501704755832540c23f702213, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731310233163 2024-11-11T07:30:35,619 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aab4aeb8ab7405eadfcbcbbe6b47f68, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731310233163 2024-11-11T07:30:35,619 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1444c34086274c18b7425ee010d3714e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731310233897 2024-11-11T07:30:35,619 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting a1f9d5acba87405889d20aca2c2f10f0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731310233897 2024-11-11T07:30:35,631 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#B#compaction#290 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:35,632 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/589c1e5b056245c7bedaec5102b3fa05 is 50, key is test_row_0/B:col10/1731310233897/Put/seqid=0 2024-11-11T07:30:35,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-11T07:30:35,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:35,633 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T07:30:35,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:35,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:35,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:35,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:35,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:35,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:35,640 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. as already flushing 2024-11-11T07:30:35,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:35,641 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#A#compaction#291 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:35,642 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/725d220667f3425ea4589bd44e40e364 is 50, key is test_row_0/A:col10/1731310233897/Put/seqid=0 2024-11-11T07:30:35,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/d90d884e1fcc4dc485f8ce2dc21978c1 is 50, key is test_row_0/A:col10/1731310234337/Put/seqid=0 2024-11-11T07:30:35,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742157_1333 (size=13493) 2024-11-11T07:30:35,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742156_1332 (size=13493) 2024-11-11T07:30:35,685 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/725d220667f3425ea4589bd44e40e364 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/725d220667f3425ea4589bd44e40e364 2024-11-11T07:30:35,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:35,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310295677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:35,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:35,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310295679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:35,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:35,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:35,688 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:35,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:35,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:35,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:35,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:35,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310295687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:35,696 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:35,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742158_1334 (size=12301) 2024-11-11T07:30:35,704 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/A of 385162ecea32f0e101e7709026aea910 into 725d220667f3425ea4589bd44e40e364(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:35,704 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:35,705 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/A, priority=13, startTime=1731310235614; duration=0sec 2024-11-11T07:30:35,705 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:35,705 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:A 2024-11-11T07:30:35,705 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:35,705 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:35,706 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/C is initiating minor compaction (all files) 2024-11-11T07:30:35,706 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/C in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:35,706 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a903f6db85e34e8781fb8eac35ec7e80, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a2c3b883c4f341beb410272e3375417b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/854447b9541c41daa264df39c61a732d] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=37.1 K 2024-11-11T07:30:35,706 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting a903f6db85e34e8781fb8eac35ec7e80, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=460, earliestPutTs=1731310232439 2024-11-11T07:30:35,706 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2c3b883c4f341beb410272e3375417b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=480, earliestPutTs=1731310233163 2024-11-11T07:30:35,706 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 854447b9541c41daa264df39c61a732d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731310233897 2024-11-11T07:30:35,733 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#C#compaction#293 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:35,733 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/79b50b29c43b4375b9a8508b9eee92c5 is 50, key is test_row_0/C:col10/1731310233897/Put/seqid=0 2024-11-11T07:30:35,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742159_1335 (size=13459) 2024-11-11T07:30:35,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310295795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:35,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:35,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310295795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:35,799 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:35,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:35,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310295805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:35,806 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:36,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310296004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:36,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310296004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,007 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:36,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:36,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310296015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:36,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,019 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:36,089 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/589c1e5b056245c7bedaec5102b3fa05 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/589c1e5b056245c7bedaec5102b3fa05 2024-11-11T07:30:36,096 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/B of 385162ecea32f0e101e7709026aea910 into 589c1e5b056245c7bedaec5102b3fa05(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:36,096 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:36,096 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/B, priority=13, startTime=1731310235614; duration=0sec 2024-11-11T07:30:36,096 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:36,096 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:B 2024-11-11T07:30:36,104 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/d90d884e1fcc4dc485f8ce2dc21978c1 2024-11-11T07:30:36,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/1a5a3e0269ea426dadcc798bdf01e31c is 50, key is test_row_0/B:col10/1731310234337/Put/seqid=0 2024-11-11T07:30:36,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742160_1336 (size=12301) 2024-11-11T07:30:36,183 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/79b50b29c43b4375b9a8508b9eee92c5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/79b50b29c43b4375b9a8508b9eee92c5 2024-11-11T07:30:36,195 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/C of 385162ecea32f0e101e7709026aea910 into 79b50b29c43b4375b9a8508b9eee92c5(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:36,195 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:36,195 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/C, priority=13, startTime=1731310235614; duration=0sec 2024-11-11T07:30:36,195 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:36,195 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:C 2024-11-11T07:30:36,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310296314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,317 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:36,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310296314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,318 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:36,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:36,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310296324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:36,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T07:30:36,516 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:36,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 213 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49976 deadline: 1731310296514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:36,516 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_1' in 385162ecea32f0e101e7709026aea910 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,517 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:36,527 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/1a5a3e0269ea426dadcc798bdf01e31c 2024-11-11T07:30:36,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/e4da2556f31646a394bd9a4f7f09ca32 is 50, key is test_row_0/C:col10/1731310234337/Put/seqid=0 2024-11-11T07:30:36,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742161_1337 (size=12301) 2024-11-11T07:30:36,575 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=513 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/e4da2556f31646a394bd9a4f7f09ca32 2024-11-11T07:30:36,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/d90d884e1fcc4dc485f8ce2dc21978c1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/d90d884e1fcc4dc485f8ce2dc21978c1 2024-11-11T07:30:36,588 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/d90d884e1fcc4dc485f8ce2dc21978c1, entries=150, sequenceid=513, filesize=12.0 K 2024-11-11T07:30:36,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/1a5a3e0269ea426dadcc798bdf01e31c as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/1a5a3e0269ea426dadcc798bdf01e31c 2024-11-11T07:30:36,596 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/1a5a3e0269ea426dadcc798bdf01e31c, entries=150, sequenceid=513, filesize=12.0 K 2024-11-11T07:30:36,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/e4da2556f31646a394bd9a4f7f09ca32 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e4da2556f31646a394bd9a4f7f09ca32 2024-11-11T07:30:36,604 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e4da2556f31646a394bd9a4f7f09ca32, entries=150, sequenceid=513, filesize=12.0 K 2024-11-11T07:30:36,605 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 385162ecea32f0e101e7709026aea910 in 972ms, sequenceid=513, compaction requested=false 2024-11-11T07:30:36,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:36,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:36,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-11T07:30:36,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-11T07:30:36,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-11T07:30:36,608 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4000 sec 2024-11-11T07:30:36,610 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.4180 sec 2024-11-11T07:30:36,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T07:30:36,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:36,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:36,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:36,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:36,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:36,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:36,835 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:36,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/b40e7681afe34ec1a3695672201d55c2 is 50, key is test_row_0/A:col10/1731310236831/Put/seqid=0 2024-11-11T07:30:36,869 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:36,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310296864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:36,874 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:36,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310296869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:36,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:36,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:36,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310296870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:36,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,876 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:36,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742162_1338 (size=17181) 2024-11-11T07:30:36,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/b40e7681afe34ec1a3695672201d55c2 2024-11-11T07:30:36,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/2be711fab4464ea4a609836142b115dc is 50, key is test_row_0/B:col10/1731310236831/Put/seqid=0 2024-11-11T07:30:36,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742163_1339 (size=12301) 2024-11-11T07:30:36,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:36,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310296977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:36,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:36,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310296985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:36,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,988 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:36,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:36,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310296985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:36,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:36,989 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:37,175 INFO [Thread-1081 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1081. 2024-11-11T07:30:37,176 DEBUG [Thread-1081 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:37,176 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:37,176 DEBUG [Thread-1081 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:37,176 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:37,176 INFO [Thread-1077 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1077. 2024-11-11T07:30:37,176 DEBUG [Thread-1077 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:37,176 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:37,176 DEBUG [Thread-1077 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:37,178 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:37,179 INFO [Thread-1085 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1085. 2024-11-11T07:30:37,179 DEBUG [Thread-1085 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:37,179 DEBUG [Thread-1085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:37,179 DEBUG [Thread-1085 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:37,182 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:37,183 INFO [Thread-1083 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1083. 2024-11-11T07:30:37,183 DEBUG [Thread-1083 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:37,183 DEBUG [Thread-1083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:37,183 DEBUG [Thread-1083 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:37,183 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:37,183 INFO [Thread-1079 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1079. 2024-11-11T07:30:37,183 DEBUG [Thread-1079 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:37,183 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:37,183 DEBUG [Thread-1079 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:37,184 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:37,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:37,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:37,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310297194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310297194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:37,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310297194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:37,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:37,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/2be711fab4464ea4a609836142b115dc 2024-11-11T07:30:37,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/e533d020bfe64a40998d81940dfb83ae is 50, key is test_row_0/C:col10/1731310236831/Put/seqid=0 2024-11-11T07:30:37,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742164_1340 (size=12301) 2024-11-11T07:30:37,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:37,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:37,505 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49926 deadline: 1731310297505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49974 deadline: 1731310297505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:37,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:49964 deadline: 1731310297505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:30:37,748 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/e533d020bfe64a40998d81940dfb83ae 2024-11-11T07:30:37,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/b40e7681afe34ec1a3695672201d55c2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/b40e7681afe34ec1a3695672201d55c2 2024-11-11T07:30:37,756 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/b40e7681afe34ec1a3695672201d55c2, entries=250, sequenceid=534, filesize=16.8 K 2024-11-11T07:30:37,757 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/2be711fab4464ea4a609836142b115dc as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/2be711fab4464ea4a609836142b115dc 2024-11-11T07:30:37,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/2be711fab4464ea4a609836142b115dc, entries=150, sequenceid=534, filesize=12.0 K 2024-11-11T07:30:37,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/e533d020bfe64a40998d81940dfb83ae as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e533d020bfe64a40998d81940dfb83ae 2024-11-11T07:30:37,765 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e533d020bfe64a40998d81940dfb83ae, entries=150, sequenceid=534, filesize=12.0 K 2024-11-11T07:30:37,766 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 385162ecea32f0e101e7709026aea910 in 933ms, sequenceid=534, compaction requested=true 2024-11-11T07:30:37,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:37,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:37,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:37,766 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:37,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:37,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:37,766 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:37,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 385162ecea32f0e101e7709026aea910:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:37,767 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:37,768 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38095 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:37,768 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/B is initiating minor compaction (all files) 2024-11-11T07:30:37,769 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42975 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:37,769 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/B in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:37,769 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/A is initiating minor compaction (all files) 2024-11-11T07:30:37,769 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/589c1e5b056245c7bedaec5102b3fa05, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/1a5a3e0269ea426dadcc798bdf01e31c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/2be711fab4464ea4a609836142b115dc] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=37.2 K 2024-11-11T07:30:37,769 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/A in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:37,769 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/725d220667f3425ea4589bd44e40e364, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/d90d884e1fcc4dc485f8ce2dc21978c1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/b40e7681afe34ec1a3695672201d55c2] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=42.0 K 2024-11-11T07:30:37,769 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 589c1e5b056245c7bedaec5102b3fa05, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731310233897 2024-11-11T07:30:37,769 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 725d220667f3425ea4589bd44e40e364, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731310233897 2024-11-11T07:30:37,769 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a5a3e0269ea426dadcc798bdf01e31c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1731310234328 2024-11-11T07:30:37,770 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting d90d884e1fcc4dc485f8ce2dc21978c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1731310234328 2024-11-11T07:30:37,770 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 2be711fab4464ea4a609836142b115dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731310235672 2024-11-11T07:30:37,770 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting b40e7681afe34ec1a3695672201d55c2, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731310235672 2024-11-11T07:30:37,780 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#B#compaction#299 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:37,781 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/d224b99d20324371ab7caf8ed6174da2 is 50, key is test_row_0/B:col10/1731310236831/Put/seqid=0 2024-11-11T07:30:37,810 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#A#compaction#300 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:37,811 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/69a71caf91a844629cdf1d7bfc656e3b is 50, key is test_row_0/A:col10/1731310236831/Put/seqid=0 2024-11-11T07:30:37,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742165_1341 (size=13595) 2024-11-11T07:30:37,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742166_1342 (size=13595) 2024-11-11T07:30:37,837 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/69a71caf91a844629cdf1d7bfc656e3b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/69a71caf91a844629cdf1d7bfc656e3b 2024-11-11T07:30:37,843 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/A of 385162ecea32f0e101e7709026aea910 into 69a71caf91a844629cdf1d7bfc656e3b(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:37,843 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:37,843 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/A, priority=13, startTime=1731310237766; duration=0sec 2024-11-11T07:30:37,843 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:37,843 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:A 2024-11-11T07:30:37,843 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:37,844 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:37,844 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 385162ecea32f0e101e7709026aea910/C is initiating minor compaction (all files) 2024-11-11T07:30:37,844 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 385162ecea32f0e101e7709026aea910/C in TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:37,845 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/79b50b29c43b4375b9a8508b9eee92c5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e4da2556f31646a394bd9a4f7f09ca32, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e533d020bfe64a40998d81940dfb83ae] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp, totalSize=37.2 K 2024-11-11T07:30:37,845 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 79b50b29c43b4375b9a8508b9eee92c5, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=496, earliestPutTs=1731310233897 2024-11-11T07:30:37,845 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4da2556f31646a394bd9a4f7f09ca32, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=513, earliestPutTs=1731310234328 2024-11-11T07:30:37,846 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting e533d020bfe64a40998d81940dfb83ae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731310235672 2024-11-11T07:30:37,891 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 385162ecea32f0e101e7709026aea910#C#compaction#301 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:37,892 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/0bb82de4d36f411b98289f52cfaf749f is 50, key is test_row_0/C:col10/1731310236831/Put/seqid=0 2024-11-11T07:30:37,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742167_1343 (size=13561) 2024-11-11T07:30:37,905 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/0bb82de4d36f411b98289f52cfaf749f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/0bb82de4d36f411b98289f52cfaf749f 2024-11-11T07:30:37,911 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/C of 385162ecea32f0e101e7709026aea910 into 0bb82de4d36f411b98289f52cfaf749f(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:37,911 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:37,911 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/C, priority=13, startTime=1731310237766; duration=0sec 2024-11-11T07:30:37,911 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:37,911 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:C 2024-11-11T07:30:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:38,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T07:30:38,023 INFO [Thread-1068 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1068. 2024-11-11T07:30:38,023 DEBUG [Thread-1068 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:38,023 DEBUG [Thread-1068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:38,023 DEBUG [Thread-1068 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:38,023 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:38,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:38,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:38,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:38,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:38,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:38,027 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:38,028 INFO [Thread-1070 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1070. 2024-11-11T07:30:38,028 DEBUG [Thread-1070 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:38,028 DEBUG [Thread-1070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:38,028 DEBUG [Thread-1070 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:38,028 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:38,033 INFO [Thread-1074 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1074. 2024-11-11T07:30:38,033 DEBUG [Thread-1074 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:38,033 DEBUG [Thread-1074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:38,033 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:38,033 DEBUG [Thread-1074 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:38,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/c1c3f757f52d41f1a91d0eef652fdbc7 is 50, key is test_row_0/A:col10/1731310238019/Put/seqid=0 2024-11-11T07:30:38,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742168_1344 (size=12301) 2024-11-11T07:30:38,045 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/c1c3f757f52d41f1a91d0eef652fdbc7 2024-11-11T07:30:38,057 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/e615baedda9844ae875e66942a2dccc8 is 50, key is test_row_0/B:col10/1731310238019/Put/seqid=0 2024-11-11T07:30:38,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742169_1345 (size=12301) 2024-11-11T07:30:38,066 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/e615baedda9844ae875e66942a2dccc8 2024-11-11T07:30:38,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/789b93f6c32b41d9a5fa61f054e0edd6 is 50, key is test_row_0/C:col10/1731310238019/Put/seqid=0 2024-11-11T07:30:38,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742170_1346 (size=12301) 2024-11-11T07:30:38,231 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/d224b99d20324371ab7caf8ed6174da2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/d224b99d20324371ab7caf8ed6174da2 2024-11-11T07:30:38,237 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 385162ecea32f0e101e7709026aea910/B of 385162ecea32f0e101e7709026aea910 into d224b99d20324371ab7caf8ed6174da2(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:38,237 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:38,237 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., storeName=385162ecea32f0e101e7709026aea910/B, priority=13, startTime=1731310237766; duration=0sec 2024-11-11T07:30:38,238 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:38,238 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 385162ecea32f0e101e7709026aea910:B 2024-11-11T07:30:38,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-11T07:30:38,365 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:38,490 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=552 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/789b93f6c32b41d9a5fa61f054e0edd6 2024-11-11T07:30:38,494 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/c1c3f757f52d41f1a91d0eef652fdbc7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/c1c3f757f52d41f1a91d0eef652fdbc7 2024-11-11T07:30:38,501 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/c1c3f757f52d41f1a91d0eef652fdbc7, entries=150, sequenceid=552, filesize=12.0 K 2024-11-11T07:30:38,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/e615baedda9844ae875e66942a2dccc8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/e615baedda9844ae875e66942a2dccc8 2024-11-11T07:30:38,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/e615baedda9844ae875e66942a2dccc8, entries=150, sequenceid=552, filesize=12.0 K 2024-11-11T07:30:38,507 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/789b93f6c32b41d9a5fa61f054e0edd6 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/789b93f6c32b41d9a5fa61f054e0edd6 2024-11-11T07:30:38,514 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/789b93f6c32b41d9a5fa61f054e0edd6, entries=150, sequenceid=552, filesize=12.0 K 2024-11-11T07:30:38,515 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=6.71 KB/6870 for 385162ecea32f0e101e7709026aea910 in 493ms, sequenceid=552, compaction requested=false 2024-11-11T07:30:38,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 385162ecea32f0e101e7709026aea910: 2024-11-11T07:30:38,536 INFO [Thread-1072 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1072. 2024-11-11T07:30:38,536 DEBUG [Thread-1072 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:38,536 DEBUG [Thread-1072 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:38,536 DEBUG [Thread-1072 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:38,536 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:39,416 INFO [Thread-1066 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1066. 2024-11-11T07:30:39,416 DEBUG [Thread-1066 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:30:39,416 DEBUG [Thread-1066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:39,416 DEBUG [Thread-1066 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:39,416 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:39,416 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T07:30:39,416 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-11T07:30:39,416 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-11T07:30:39,416 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-11T07:30:39,416 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 144 2024-11-11T07:30:39,416 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-11T07:30:39,416 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T07:30:39,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1981 2024-11-11T07:30:39,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1931 2024-11-11T07:30:39,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1987 2024-11-11T07:30:39,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1941 2024-11-11T07:30:39,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1937 2024-11-11T07:30:39,417 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T07:30:39,417 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T07:30:39,417 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:30:39,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:39,417 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:39,417 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:30:39,418 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T07:30:39,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:39,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T07:30:39,421 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310239421"}]},"ts":"1731310239421"} 2024-11-11T07:30:39,423 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T07:30:39,423 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T07:30:39,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:30:39,425 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=385162ecea32f0e101e7709026aea910, UNASSIGN}] 2024-11-11T07:30:39,426 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=385162ecea32f0e101e7709026aea910, UNASSIGN 2024-11-11T07:30:39,426 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=385162ecea32f0e101e7709026aea910, regionState=CLOSING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:39,428 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=385162ecea32f0e101e7709026aea910, UNASSIGN because future has completed 2024-11-11T07:30:39,428 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:30:39,428 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:30:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T07:30:39,580 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:39,580 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:30:39,580 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 385162ecea32f0e101e7709026aea910, disabling compactions & flushes 2024-11-11T07:30:39,580 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:39,580 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:39,581 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. after waiting 1 ms 2024-11-11T07:30:39,581 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:39,581 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 385162ecea32f0e101e7709026aea910 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-11T07:30:39,581 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=A 2024-11-11T07:30:39,581 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:39,581 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=B 2024-11-11T07:30:39,581 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:39,581 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 385162ecea32f0e101e7709026aea910, store=C 2024-11-11T07:30:39,581 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:39,585 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/e982a4b1dbc04485b40202e8b07e759d is 50, key is test_row_1/A:col10/1731310239415/Put/seqid=0 2024-11-11T07:30:39,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742171_1347 (size=9857) 2024-11-11T07:30:39,594 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/e982a4b1dbc04485b40202e8b07e759d 2024-11-11T07:30:39,600 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/f69fb1de231e43639194d3f7e1bd1c4f is 50, key is test_row_1/B:col10/1731310239415/Put/seqid=0 2024-11-11T07:30:39,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742172_1348 (size=9857) 2024-11-11T07:30:39,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T07:30:40,004 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/f69fb1de231e43639194d3f7e1bd1c4f 2024-11-11T07:30:40,011 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/aed69c44ca5b4694a58493a888024859 is 50, key is test_row_1/C:col10/1731310239415/Put/seqid=0 2024-11-11T07:30:40,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742173_1349 (size=9857) 2024-11-11T07:30:40,023 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/aed69c44ca5b4694a58493a888024859 2024-11-11T07:30:40,027 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/A/e982a4b1dbc04485b40202e8b07e759d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/e982a4b1dbc04485b40202e8b07e759d 2024-11-11T07:30:40,031 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/e982a4b1dbc04485b40202e8b07e759d, entries=100, sequenceid=559, filesize=9.6 K 2024-11-11T07:30:40,032 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/B/f69fb1de231e43639194d3f7e1bd1c4f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f69fb1de231e43639194d3f7e1bd1c4f 2024-11-11T07:30:40,037 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f69fb1de231e43639194d3f7e1bd1c4f, entries=100, sequenceid=559, filesize=9.6 K 2024-11-11T07:30:40,038 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/.tmp/C/aed69c44ca5b4694a58493a888024859 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/aed69c44ca5b4694a58493a888024859 2024-11-11T07:30:40,042 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/aed69c44ca5b4694a58493a888024859, entries=100, sequenceid=559, filesize=9.6 K 2024-11-11T07:30:40,043 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 385162ecea32f0e101e7709026aea910 in 462ms, sequenceid=559, compaction requested=true 2024-11-11T07:30:40,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/edabf3d46e894cc792cfbc3e65580ec0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/5fb6095ddd98453f8f3dc2e06af3b778, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/3930a3c12d4643ccbc44da7869934d46, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/64614399dfee4662965bc74dce12c5ff, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/a195c28df8494a568744efc561dc1d96, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/6e626442fbf04bb7b2f0533c94f2e672, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/6975f93300a0450b89bac20cd0f365a4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/0652d6ea098749c1adf34451782cf190, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/3237a58875854b78b7321266058d7425, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/d0ec91f49efe4990ab8b6f9a4d25ccf5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/2ab6035de7a346dd88b04aba76f648cc, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/e6583b69320e4191956400c86d6e6157, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9fcb95b6ec984115b10fefe4cf5bc615, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/11becc0501dc49b58a9506e4ba3deb68, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9fc4f6ed4431419ba15bc9b7ee639736, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/013fb53ecb39425a87633c97ace8fc9d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/6850c79d91fb44beb626e0de9303b4a8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9544dc96841943608140977ab3060c93, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/19edb9b1c33147bc8ea144fa15075f87, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/8a2ad4117fee46e98b46e5515ba11621, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9b6318133d324148abb4de9403a5b07d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9e0b4aa19b3241e4a6dd969205bcb60c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1424d29836b54cd7b27a5a9308ec1fcb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/a630791b540d4ea7b4923f5547905cf5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1bc3ed91174e4b7c8cfa4c34262e2dde, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/5c3890646b62463796fbb7bb39c35f28, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1353829d62c94225ad887908d5673936, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/4487442c570849e89ef341342568a978, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/b9afd3b141af47fa8aff0c79e815b1b4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/03607a5b1d7a44898a5567d76f8fd15d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9966664370da4781bb2989059e9f4ec7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/138b56b6e446497083996c70c8817011, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/cfd9757a89d248d49f859a330a7d76e2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/39a2f59501704755832540c23f702213, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/725d220667f3425ea4589bd44e40e364, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1444c34086274c18b7425ee010d3714e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/d90d884e1fcc4dc485f8ce2dc21978c1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/b40e7681afe34ec1a3695672201d55c2] to archive 2024-11-11T07:30:40,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T07:30:40,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:30:40,046 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/edabf3d46e894cc792cfbc3e65580ec0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/edabf3d46e894cc792cfbc3e65580ec0 2024-11-11T07:30:40,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/5fb6095ddd98453f8f3dc2e06af3b778 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/5fb6095ddd98453f8f3dc2e06af3b778 2024-11-11T07:30:40,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/3930a3c12d4643ccbc44da7869934d46 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/3930a3c12d4643ccbc44da7869934d46 2024-11-11T07:30:40,050 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/64614399dfee4662965bc74dce12c5ff to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/64614399dfee4662965bc74dce12c5ff 2024-11-11T07:30:40,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/a195c28df8494a568744efc561dc1d96 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/a195c28df8494a568744efc561dc1d96 2024-11-11T07:30:40,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/6e626442fbf04bb7b2f0533c94f2e672 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/6e626442fbf04bb7b2f0533c94f2e672 2024-11-11T07:30:40,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/6975f93300a0450b89bac20cd0f365a4 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/6975f93300a0450b89bac20cd0f365a4 2024-11-11T07:30:40,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/0652d6ea098749c1adf34451782cf190 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/0652d6ea098749c1adf34451782cf190 2024-11-11T07:30:40,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/3237a58875854b78b7321266058d7425 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/3237a58875854b78b7321266058d7425 2024-11-11T07:30:40,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/d0ec91f49efe4990ab8b6f9a4d25ccf5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/d0ec91f49efe4990ab8b6f9a4d25ccf5 2024-11-11T07:30:40,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/2ab6035de7a346dd88b04aba76f648cc to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/2ab6035de7a346dd88b04aba76f648cc 2024-11-11T07:30:40,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/e6583b69320e4191956400c86d6e6157 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/e6583b69320e4191956400c86d6e6157 2024-11-11T07:30:40,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9fcb95b6ec984115b10fefe4cf5bc615 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9fcb95b6ec984115b10fefe4cf5bc615 2024-11-11T07:30:40,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/11becc0501dc49b58a9506e4ba3deb68 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/11becc0501dc49b58a9506e4ba3deb68 2024-11-11T07:30:40,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9fc4f6ed4431419ba15bc9b7ee639736 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9fc4f6ed4431419ba15bc9b7ee639736 2024-11-11T07:30:40,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/013fb53ecb39425a87633c97ace8fc9d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/013fb53ecb39425a87633c97ace8fc9d 2024-11-11T07:30:40,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/6850c79d91fb44beb626e0de9303b4a8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/6850c79d91fb44beb626e0de9303b4a8 2024-11-11T07:30:40,065 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9544dc96841943608140977ab3060c93 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9544dc96841943608140977ab3060c93 2024-11-11T07:30:40,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/19edb9b1c33147bc8ea144fa15075f87 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/19edb9b1c33147bc8ea144fa15075f87 2024-11-11T07:30:40,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/8a2ad4117fee46e98b46e5515ba11621 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/8a2ad4117fee46e98b46e5515ba11621 2024-11-11T07:30:40,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9b6318133d324148abb4de9403a5b07d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9b6318133d324148abb4de9403a5b07d 2024-11-11T07:30:40,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9e0b4aa19b3241e4a6dd969205bcb60c to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9e0b4aa19b3241e4a6dd969205bcb60c 2024-11-11T07:30:40,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1424d29836b54cd7b27a5a9308ec1fcb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1424d29836b54cd7b27a5a9308ec1fcb 2024-11-11T07:30:40,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/a630791b540d4ea7b4923f5547905cf5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/a630791b540d4ea7b4923f5547905cf5 2024-11-11T07:30:40,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1bc3ed91174e4b7c8cfa4c34262e2dde to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1bc3ed91174e4b7c8cfa4c34262e2dde 2024-11-11T07:30:40,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/5c3890646b62463796fbb7bb39c35f28 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/5c3890646b62463796fbb7bb39c35f28 2024-11-11T07:30:40,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1353829d62c94225ad887908d5673936 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1353829d62c94225ad887908d5673936 2024-11-11T07:30:40,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/4487442c570849e89ef341342568a978 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/4487442c570849e89ef341342568a978 2024-11-11T07:30:40,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/b9afd3b141af47fa8aff0c79e815b1b4 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/b9afd3b141af47fa8aff0c79e815b1b4 2024-11-11T07:30:40,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/03607a5b1d7a44898a5567d76f8fd15d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/03607a5b1d7a44898a5567d76f8fd15d 2024-11-11T07:30:40,079 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9966664370da4781bb2989059e9f4ec7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/9966664370da4781bb2989059e9f4ec7 2024-11-11T07:30:40,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/138b56b6e446497083996c70c8817011 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/138b56b6e446497083996c70c8817011 2024-11-11T07:30:40,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/cfd9757a89d248d49f859a330a7d76e2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/cfd9757a89d248d49f859a330a7d76e2 2024-11-11T07:30:40,082 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/39a2f59501704755832540c23f702213 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/39a2f59501704755832540c23f702213 2024-11-11T07:30:40,083 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/725d220667f3425ea4589bd44e40e364 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/725d220667f3425ea4589bd44e40e364 2024-11-11T07:30:40,084 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1444c34086274c18b7425ee010d3714e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/1444c34086274c18b7425ee010d3714e 2024-11-11T07:30:40,085 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/d90d884e1fcc4dc485f8ce2dc21978c1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/d90d884e1fcc4dc485f8ce2dc21978c1 2024-11-11T07:30:40,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/b40e7681afe34ec1a3695672201d55c2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/b40e7681afe34ec1a3695672201d55c2 2024-11-11T07:30:40,088 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f8a602eb2d15411d9caf4fa69732509a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f57b0cdc346b4b6b80abc1ca81732b43, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/7845ea38663546bfa442860a0a784dfd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/25a46c73df9c445f8ee8a72eb1e3469a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/7ed42671330e49c6bc8562780c6c563c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5fe4c092e3214d2e98fbdac2a6db4ba1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b900c33dc2b24dd397cae156489293e1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/83176f4f34bf495c9d1cf2dbf47ea9fb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/1b6d6ecdfd7c477b878202c73c6d667f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/4e7e45236a4744059ace6a8fae865c5a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/aaeccd367e26437abc270e4a301fb403, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/bdb907ddf85144db9da87c120140b710, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/fa4cdc36e95c4539b793fee9c7e04827, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b1073c6ff2e84f06965601c731bd6805, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/fa19a709baa247648b4ff654b3f76fcb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/4412a911f5634d18834ff3cbc5e6f292, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/24661ec424ec4f738aec6e741411ad35, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/8ab5842218e34a6389bbeb0a7eeb00e2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5760b309871049ee8b1e2b4d28cf0494, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/931f7d1409474051a0df72fba4e62ff5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/905677c3e2e5480e89ed112bc43ca445, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/045dea5b474a470ab658dcbe78b6ecc8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/30c3492585ac4c058591d69769ff2da6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/de3c06c1df1446539fb4601909e8c614, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/c94594db10c747019ffb157e95d219a3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f34c9d54a7aa44c8acbced2029a8a07f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/11755c2066e4438684163cf74c0dcfc8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b8f6f966367c41daa7414a8747df1ccb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/67db11d5662948f5a20eceb30068a6dd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f4a63c4f2db0495db3598dcfab130911, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/cc4a90225b2d42d6a86f4c8830745d9f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/536b0927b8b54f7ebb6b6ab60c8d35d5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/8261837467604448a57e0a2945c9163b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5aab4aeb8ab7405eadfcbcbbe6b47f68, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/589c1e5b056245c7bedaec5102b3fa05, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/a1f9d5acba87405889d20aca2c2f10f0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/1a5a3e0269ea426dadcc798bdf01e31c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/2be711fab4464ea4a609836142b115dc] to archive 2024-11-11T07:30:40,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:30:40,090 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f8a602eb2d15411d9caf4fa69732509a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f8a602eb2d15411d9caf4fa69732509a 2024-11-11T07:30:40,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f57b0cdc346b4b6b80abc1ca81732b43 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f57b0cdc346b4b6b80abc1ca81732b43 2024-11-11T07:30:40,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/7845ea38663546bfa442860a0a784dfd to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/7845ea38663546bfa442860a0a784dfd 2024-11-11T07:30:40,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/25a46c73df9c445f8ee8a72eb1e3469a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/25a46c73df9c445f8ee8a72eb1e3469a 2024-11-11T07:30:40,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/7ed42671330e49c6bc8562780c6c563c to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/7ed42671330e49c6bc8562780c6c563c 2024-11-11T07:30:40,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5fe4c092e3214d2e98fbdac2a6db4ba1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5fe4c092e3214d2e98fbdac2a6db4ba1 2024-11-11T07:30:40,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b900c33dc2b24dd397cae156489293e1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b900c33dc2b24dd397cae156489293e1 2024-11-11T07:30:40,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/83176f4f34bf495c9d1cf2dbf47ea9fb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/83176f4f34bf495c9d1cf2dbf47ea9fb 2024-11-11T07:30:40,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/1b6d6ecdfd7c477b878202c73c6d667f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/1b6d6ecdfd7c477b878202c73c6d667f 2024-11-11T07:30:40,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/4e7e45236a4744059ace6a8fae865c5a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/4e7e45236a4744059ace6a8fae865c5a 2024-11-11T07:30:40,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/aaeccd367e26437abc270e4a301fb403 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/aaeccd367e26437abc270e4a301fb403 2024-11-11T07:30:40,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/bdb907ddf85144db9da87c120140b710 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/bdb907ddf85144db9da87c120140b710 2024-11-11T07:30:40,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/fa4cdc36e95c4539b793fee9c7e04827 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/fa4cdc36e95c4539b793fee9c7e04827 2024-11-11T07:30:40,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b1073c6ff2e84f06965601c731bd6805 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b1073c6ff2e84f06965601c731bd6805 2024-11-11T07:30:40,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/fa19a709baa247648b4ff654b3f76fcb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/fa19a709baa247648b4ff654b3f76fcb 2024-11-11T07:30:40,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/4412a911f5634d18834ff3cbc5e6f292 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/4412a911f5634d18834ff3cbc5e6f292 2024-11-11T07:30:40,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/24661ec424ec4f738aec6e741411ad35 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/24661ec424ec4f738aec6e741411ad35 2024-11-11T07:30:40,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/8ab5842218e34a6389bbeb0a7eeb00e2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/8ab5842218e34a6389bbeb0a7eeb00e2 2024-11-11T07:30:40,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5760b309871049ee8b1e2b4d28cf0494 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5760b309871049ee8b1e2b4d28cf0494 2024-11-11T07:30:40,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/931f7d1409474051a0df72fba4e62ff5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/931f7d1409474051a0df72fba4e62ff5 2024-11-11T07:30:40,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/905677c3e2e5480e89ed112bc43ca445 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/905677c3e2e5480e89ed112bc43ca445 2024-11-11T07:30:40,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/045dea5b474a470ab658dcbe78b6ecc8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/045dea5b474a470ab658dcbe78b6ecc8 2024-11-11T07:30:40,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/30c3492585ac4c058591d69769ff2da6 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/30c3492585ac4c058591d69769ff2da6 2024-11-11T07:30:40,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/de3c06c1df1446539fb4601909e8c614 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/de3c06c1df1446539fb4601909e8c614 2024-11-11T07:30:40,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/c94594db10c747019ffb157e95d219a3 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/c94594db10c747019ffb157e95d219a3 2024-11-11T07:30:40,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f34c9d54a7aa44c8acbced2029a8a07f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f34c9d54a7aa44c8acbced2029a8a07f 2024-11-11T07:30:40,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/11755c2066e4438684163cf74c0dcfc8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/11755c2066e4438684163cf74c0dcfc8 2024-11-11T07:30:40,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b8f6f966367c41daa7414a8747df1ccb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/b8f6f966367c41daa7414a8747df1ccb 2024-11-11T07:30:40,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/67db11d5662948f5a20eceb30068a6dd to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/67db11d5662948f5a20eceb30068a6dd 2024-11-11T07:30:40,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f4a63c4f2db0495db3598dcfab130911 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f4a63c4f2db0495db3598dcfab130911 2024-11-11T07:30:40,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/cc4a90225b2d42d6a86f4c8830745d9f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/cc4a90225b2d42d6a86f4c8830745d9f 2024-11-11T07:30:40,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/536b0927b8b54f7ebb6b6ab60c8d35d5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/536b0927b8b54f7ebb6b6ab60c8d35d5 2024-11-11T07:30:40,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/8261837467604448a57e0a2945c9163b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/8261837467604448a57e0a2945c9163b 2024-11-11T07:30:40,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5aab4aeb8ab7405eadfcbcbbe6b47f68 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/5aab4aeb8ab7405eadfcbcbbe6b47f68 2024-11-11T07:30:40,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/589c1e5b056245c7bedaec5102b3fa05 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/589c1e5b056245c7bedaec5102b3fa05 2024-11-11T07:30:40,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/a1f9d5acba87405889d20aca2c2f10f0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/a1f9d5acba87405889d20aca2c2f10f0 2024-11-11T07:30:40,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/1a5a3e0269ea426dadcc798bdf01e31c to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/1a5a3e0269ea426dadcc798bdf01e31c 2024-11-11T07:30:40,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/2be711fab4464ea4a609836142b115dc to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/2be711fab4464ea4a609836142b115dc 2024-11-11T07:30:40,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/adf43b21beb24c3a8d86a505fc4747a0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e9e0d1efb0cb4c5695cccffdc2c84eb8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/984395307849466188682aa9834674dd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/2a4cc419861a499281d56f02f2bd1329, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/c0d053258c42402d9694b721dc7c5bf9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/0bc0a3f2befb477ca9d02e8b9eaf2e16, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/de1fcaa86eb24a7c99a7148a1d2c52da, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/601b092f716940aa886e5784a3004154, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/642ee7782f0244af8e25eed4aeb929b3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/9de8fdd8647b4630b5aa4f4f5b8d7154, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/b13cd70f417846ab8b6d3ed714fe44b8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/f570bea7ef1844cd99425ed4ea80e012, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/96602b52926349689d943bf3c735589d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/34109cddc51e461698efc030e842c452, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a3bc54b3f64748999a4f99192dec1ed0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/3822e797d86f410496cd154cc0fb6ce8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/0bdecc321b2a469d92fcb960a68b2989, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/7312ff592b084feab029e115d613a438, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/ddb417b8c43c49c48472142537247fa2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/b1f7e85221174bec8788c47d3c11746a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/baeff82d299f4054936c6506fa0a94e2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/936fbbcebba34deda1e68076dd6c28bb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/6fbb67a8c5a1431fa7cb89617f2421df, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/cbc41f68ee9547e99235df533c5b661e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/6d21ae080f894e0287674a0f5c54c416, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/38294795e74843a2a1c84e41688491e3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/5c1e72f873f1423093bb1cfb830e96c7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/5a036da7567f40b2b1390397bb1de436, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/ae83d5ca94ef43c9817c97d79605eb7e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/279878a602da442b8295b66e50f29279, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a903f6db85e34e8781fb8eac35ec7e80, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/2b0cb89191b44b1bad5dbcad6cc69c0d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a2c3b883c4f341beb410272e3375417b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/79b50b29c43b4375b9a8508b9eee92c5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/854447b9541c41daa264df39c61a732d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e4da2556f31646a394bd9a4f7f09ca32, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e533d020bfe64a40998d81940dfb83ae] to archive 2024-11-11T07:30:40,138 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:30:40,140 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/adf43b21beb24c3a8d86a505fc4747a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/adf43b21beb24c3a8d86a505fc4747a0 2024-11-11T07:30:40,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e9e0d1efb0cb4c5695cccffdc2c84eb8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e9e0d1efb0cb4c5695cccffdc2c84eb8 2024-11-11T07:30:40,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/984395307849466188682aa9834674dd to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/984395307849466188682aa9834674dd 2024-11-11T07:30:40,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/2a4cc419861a499281d56f02f2bd1329 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/2a4cc419861a499281d56f02f2bd1329 2024-11-11T07:30:40,143 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/c0d053258c42402d9694b721dc7c5bf9 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/c0d053258c42402d9694b721dc7c5bf9 2024-11-11T07:30:40,145 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/0bc0a3f2befb477ca9d02e8b9eaf2e16 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/0bc0a3f2befb477ca9d02e8b9eaf2e16 2024-11-11T07:30:40,146 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/de1fcaa86eb24a7c99a7148a1d2c52da to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/de1fcaa86eb24a7c99a7148a1d2c52da 2024-11-11T07:30:40,147 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/601b092f716940aa886e5784a3004154 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/601b092f716940aa886e5784a3004154 2024-11-11T07:30:40,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/642ee7782f0244af8e25eed4aeb929b3 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/642ee7782f0244af8e25eed4aeb929b3 2024-11-11T07:30:40,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/9de8fdd8647b4630b5aa4f4f5b8d7154 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/9de8fdd8647b4630b5aa4f4f5b8d7154 2024-11-11T07:30:40,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/b13cd70f417846ab8b6d3ed714fe44b8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/b13cd70f417846ab8b6d3ed714fe44b8 2024-11-11T07:30:40,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/f570bea7ef1844cd99425ed4ea80e012 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/f570bea7ef1844cd99425ed4ea80e012 2024-11-11T07:30:40,153 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/96602b52926349689d943bf3c735589d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/96602b52926349689d943bf3c735589d 2024-11-11T07:30:40,154 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/34109cddc51e461698efc030e842c452 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/34109cddc51e461698efc030e842c452 2024-11-11T07:30:40,155 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a3bc54b3f64748999a4f99192dec1ed0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a3bc54b3f64748999a4f99192dec1ed0 2024-11-11T07:30:40,156 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/3822e797d86f410496cd154cc0fb6ce8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/3822e797d86f410496cd154cc0fb6ce8 2024-11-11T07:30:40,157 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/0bdecc321b2a469d92fcb960a68b2989 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/0bdecc321b2a469d92fcb960a68b2989 2024-11-11T07:30:40,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/7312ff592b084feab029e115d613a438 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/7312ff592b084feab029e115d613a438 2024-11-11T07:30:40,159 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/ddb417b8c43c49c48472142537247fa2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/ddb417b8c43c49c48472142537247fa2 2024-11-11T07:30:40,161 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/b1f7e85221174bec8788c47d3c11746a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/b1f7e85221174bec8788c47d3c11746a 2024-11-11T07:30:40,162 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/baeff82d299f4054936c6506fa0a94e2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/baeff82d299f4054936c6506fa0a94e2 2024-11-11T07:30:40,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/936fbbcebba34deda1e68076dd6c28bb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/936fbbcebba34deda1e68076dd6c28bb 2024-11-11T07:30:40,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/6fbb67a8c5a1431fa7cb89617f2421df to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/6fbb67a8c5a1431fa7cb89617f2421df 2024-11-11T07:30:40,164 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/cbc41f68ee9547e99235df533c5b661e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/cbc41f68ee9547e99235df533c5b661e 2024-11-11T07:30:40,167 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/6d21ae080f894e0287674a0f5c54c416 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/6d21ae080f894e0287674a0f5c54c416 2024-11-11T07:30:40,168 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/38294795e74843a2a1c84e41688491e3 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/38294795e74843a2a1c84e41688491e3 2024-11-11T07:30:40,169 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/5c1e72f873f1423093bb1cfb830e96c7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/5c1e72f873f1423093bb1cfb830e96c7 2024-11-11T07:30:40,170 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/5a036da7567f40b2b1390397bb1de436 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/5a036da7567f40b2b1390397bb1de436 2024-11-11T07:30:40,171 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/ae83d5ca94ef43c9817c97d79605eb7e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/ae83d5ca94ef43c9817c97d79605eb7e 2024-11-11T07:30:40,172 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/279878a602da442b8295b66e50f29279 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/279878a602da442b8295b66e50f29279 2024-11-11T07:30:40,174 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a903f6db85e34e8781fb8eac35ec7e80 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a903f6db85e34e8781fb8eac35ec7e80 2024-11-11T07:30:40,175 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/2b0cb89191b44b1bad5dbcad6cc69c0d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/2b0cb89191b44b1bad5dbcad6cc69c0d 2024-11-11T07:30:40,176 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a2c3b883c4f341beb410272e3375417b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/a2c3b883c4f341beb410272e3375417b 2024-11-11T07:30:40,177 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/79b50b29c43b4375b9a8508b9eee92c5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/79b50b29c43b4375b9a8508b9eee92c5 2024-11-11T07:30:40,179 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/854447b9541c41daa264df39c61a732d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/854447b9541c41daa264df39c61a732d 2024-11-11T07:30:40,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e4da2556f31646a394bd9a4f7f09ca32 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e4da2556f31646a394bd9a4f7f09ca32 2024-11-11T07:30:40,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e533d020bfe64a40998d81940dfb83ae to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/e533d020bfe64a40998d81940dfb83ae 2024-11-11T07:30:40,187 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/recovered.edits/562.seqid, newMaxSeqId=562, maxSeqId=1 2024-11-11T07:30:40,187 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. 2024-11-11T07:30:40,188 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 385162ecea32f0e101e7709026aea910: Waiting for close lock at 1731310239580Running coprocessor pre-close hooks at 1731310239580Disabling compacts and flushes for region at 1731310239580Disabling writes for close at 1731310239581 (+1 ms)Obtaining lock to block concurrent updates at 1731310239581Preparing flush snapshotting stores in 385162ecea32f0e101e7709026aea910 at 1731310239581Finished memstore snapshotting TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731310239581Flushing stores of TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910. at 1731310239582 (+1 ms)Flushing 385162ecea32f0e101e7709026aea910/A: creating writer at 1731310239582Flushing 385162ecea32f0e101e7709026aea910/A: appending metadata at 1731310239585 (+3 ms)Flushing 385162ecea32f0e101e7709026aea910/A: closing flushed file at 1731310239585Flushing 385162ecea32f0e101e7709026aea910/B: creating writer at 1731310239597 (+12 ms)Flushing 385162ecea32f0e101e7709026aea910/B: appending metadata at 1731310239600 (+3 ms)Flushing 385162ecea32f0e101e7709026aea910/B: closing flushed file at 1731310239600Flushing 385162ecea32f0e101e7709026aea910/C: creating writer at 1731310240008 (+408 ms)Flushing 385162ecea32f0e101e7709026aea910/C: appending metadata at 1731310240011 (+3 ms)Flushing 385162ecea32f0e101e7709026aea910/C: closing flushed file at 1731310240011Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a060852: reopening flushed file at 1731310240027 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bd8395b: reopening flushed file at 1731310240031 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ba8401c: reopening flushed file at 1731310240037 (+6 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 385162ecea32f0e101e7709026aea910 in 462ms, sequenceid=559, compaction requested=true at 1731310240043 (+6 ms)Writing region close event to WAL at 1731310240183 (+140 ms)Running coprocessor post-close hooks at 1731310240187 (+4 ms)Closed at 1731310240187 2024-11-11T07:30:40,191 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=385162ecea32f0e101e7709026aea910, regionState=CLOSED 2024-11-11T07:30:40,192 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 385162ecea32f0e101e7709026aea910 2024-11-11T07:30:40,193 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:30:40,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-11T07:30:40,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 385162ecea32f0e101e7709026aea910, server=ce2cf0244921,33345,1731310155369 in 765 msec 2024-11-11T07:30:40,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-11T07:30:40,197 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=385162ecea32f0e101e7709026aea910, UNASSIGN in 770 msec 2024-11-11T07:30:40,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-11T07:30:40,199 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 774 msec 2024-11-11T07:30:40,201 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310240200"}]},"ts":"1731310240200"} 2024-11-11T07:30:40,202 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T07:30:40,202 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T07:30:40,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 785 msec 2024-11-11T07:30:40,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-11T07:30:40,555 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:40,556 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T07:30:40,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:40,557 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:40,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T07:30:40,558 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:40,562 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910 2024-11-11T07:30:40,564 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/recovered.edits] 2024-11-11T07:30:40,567 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/69a71caf91a844629cdf1d7bfc656e3b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/69a71caf91a844629cdf1d7bfc656e3b 2024-11-11T07:30:40,568 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/c1c3f757f52d41f1a91d0eef652fdbc7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/c1c3f757f52d41f1a91d0eef652fdbc7 2024-11-11T07:30:40,569 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/e982a4b1dbc04485b40202e8b07e759d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/A/e982a4b1dbc04485b40202e8b07e759d 2024-11-11T07:30:40,571 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/d224b99d20324371ab7caf8ed6174da2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/d224b99d20324371ab7caf8ed6174da2 2024-11-11T07:30:40,572 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/e615baedda9844ae875e66942a2dccc8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/e615baedda9844ae875e66942a2dccc8 2024-11-11T07:30:40,573 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f69fb1de231e43639194d3f7e1bd1c4f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/B/f69fb1de231e43639194d3f7e1bd1c4f 2024-11-11T07:30:40,576 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/0bb82de4d36f411b98289f52cfaf749f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/0bb82de4d36f411b98289f52cfaf749f 2024-11-11T07:30:40,577 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/789b93f6c32b41d9a5fa61f054e0edd6 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/789b93f6c32b41d9a5fa61f054e0edd6 2024-11-11T07:30:40,578 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/aed69c44ca5b4694a58493a888024859 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/C/aed69c44ca5b4694a58493a888024859 2024-11-11T07:30:40,586 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/recovered.edits/562.seqid to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910/recovered.edits/562.seqid 2024-11-11T07:30:40,587 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/385162ecea32f0e101e7709026aea910 2024-11-11T07:30:40,587 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T07:30:40,591 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:40,594 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T07:30:40,597 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T07:30:40,598 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:40,598 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T07:30:40,598 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731310240598"}]},"ts":"9223372036854775807"} 2024-11-11T07:30:40,601 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T07:30:40,601 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 385162ecea32f0e101e7709026aea910, NAME => 'TestAcidGuarantees,,1731310214947.385162ecea32f0e101e7709026aea910.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T07:30:40,601 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T07:30:40,601 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731310240601"}]},"ts":"9223372036854775807"} 2024-11-11T07:30:40,603 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T07:30:40,604 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:40,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 48 msec 2024-11-11T07:30:40,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-11T07:30:40,665 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T07:30:40,665 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:40,675 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 251), OpenFileDescriptor=453 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=924 (was 926), ProcessCount=11 (was 11), AvailableMemoryMB=2499 (was 2581) 2024-11-11T07:30:40,686 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=924, ProcessCount=11, AvailableMemoryMB=2498 2024-11-11T07:30:40,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:30:40,687 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:30:40,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:40,689 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T07:30:40,690 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:40,690 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-11T07:30:40,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T07:30:40,690 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T07:30:40,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742174_1350 (size=1061) 2024-11-11T07:30:40,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T07:30:41,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T07:30:41,097 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e 2024-11-11T07:30:41,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742175_1351 (size=53) 2024-11-11T07:30:41,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T07:30:41,503 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:30:41,503 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6c7e70f5adf27c26564bc9bc160230a0, disabling compactions & flushes 2024-11-11T07:30:41,503 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:41,503 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:41,503 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. after waiting 0 ms 2024-11-11T07:30:41,503 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:41,503 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:41,503 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6c7e70f5adf27c26564bc9bc160230a0: Waiting for close lock at 1731310241503Disabling compacts and flushes for region at 1731310241503Disabling writes for close at 1731310241503Writing region close event to WAL at 1731310241503Closed at 1731310241503 2024-11-11T07:30:41,504 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T07:30:41,505 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731310241504"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731310241504"}]},"ts":"1731310241504"} 2024-11-11T07:30:41,506 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T07:30:41,507 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T07:30:41,507 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310241507"}]},"ts":"1731310241507"} 2024-11-11T07:30:41,509 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T07:30:41,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c7e70f5adf27c26564bc9bc160230a0, ASSIGN}] 2024-11-11T07:30:41,510 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c7e70f5adf27c26564bc9bc160230a0, ASSIGN 2024-11-11T07:30:41,511 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c7e70f5adf27c26564bc9bc160230a0, ASSIGN; state=OFFLINE, location=ce2cf0244921,33345,1731310155369; forceNewPlan=false, retain=false 2024-11-11T07:30:41,662 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=6c7e70f5adf27c26564bc9bc160230a0, regionState=OPENING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:41,663 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c7e70f5adf27c26564bc9bc160230a0, ASSIGN because future has completed 2024-11-11T07:30:41,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:30:41,818 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:41,819 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:30:41,819 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:41,819 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:30:41,819 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:41,819 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:41,820 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:41,821 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:30:41,822 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c7e70f5adf27c26564bc9bc160230a0 columnFamilyName A 2024-11-11T07:30:41,822 DEBUG [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:41,822 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.HStore(327): Store=6c7e70f5adf27c26564bc9bc160230a0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:30:41,822 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:41,823 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:30:41,823 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c7e70f5adf27c26564bc9bc160230a0 columnFamilyName B 2024-11-11T07:30:41,823 DEBUG [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:41,824 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.HStore(327): Store=6c7e70f5adf27c26564bc9bc160230a0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:30:41,824 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:41,824 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:30:41,825 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c7e70f5adf27c26564bc9bc160230a0 columnFamilyName C 2024-11-11T07:30:41,825 DEBUG [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:41,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T07:30:41,825 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.HStore(327): Store=6c7e70f5adf27c26564bc9bc160230a0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:30:41,825 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:41,825 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:41,826 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:41,826 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:41,827 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:41,827 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:41,828 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:30:41,828 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:41,830 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T07:30:41,830 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened 6c7e70f5adf27c26564bc9bc160230a0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70164893, jitterRate=0.04553838074207306}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:30:41,831 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:41,831 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for 6c7e70f5adf27c26564bc9bc160230a0: Running coprocessor pre-open hook at 1731310241819Writing region info on filesystem at 1731310241819Initializing all the Stores at 1731310241820 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310241820Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310241820Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310241820Cleaning up temporary data from old regions at 1731310241827 (+7 ms)Running coprocessor post-open hooks at 1731310241831 (+4 ms)Region opened successfully at 1731310241831 2024-11-11T07:30:41,832 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., pid=89, masterSystemTime=1731310241815 2024-11-11T07:30:41,834 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:41,834 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:41,834 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=6c7e70f5adf27c26564bc9bc160230a0, regionState=OPEN, openSeqNum=2, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:41,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:30:41,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-11T07:30:41,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure 6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 in 172 msec 2024-11-11T07:30:41,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-11T07:30:41,840 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c7e70f5adf27c26564bc9bc160230a0, ASSIGN in 329 msec 2024-11-11T07:30:41,841 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T07:30:41,841 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310241841"}]},"ts":"1731310241841"} 2024-11-11T07:30:41,842 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T07:30:41,843 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T07:30:41,844 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1560 sec 2024-11-11T07:30:42,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-11T07:30:42,835 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:42,836 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1af6f666, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:42,836 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:42,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:42,837 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65f8e90d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:42,838 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:42,839 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:42,840 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75fb9fb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:42,840 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:42,881 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:42,882 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56460, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:42,883 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T07:30:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is ce2cf0244921,33029,1731310154427 2024-11-11T07:30:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4fc0a3e9 2024-11-11T07:30:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T07:30:42,885 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T07:30:42,886 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:30:42,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:30:42,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T07:30:42,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742176_1352 (size=1097) 2024-11-11T07:30:43,304 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T07:30:43,304 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T07:30:43,307 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:30:43,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c7e70f5adf27c26564bc9bc160230a0, REOPEN/MOVE}] 2024-11-11T07:30:43,310 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c7e70f5adf27c26564bc9bc160230a0, REOPEN/MOVE 2024-11-11T07:30:43,311 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=6c7e70f5adf27c26564bc9bc160230a0, regionState=CLOSING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:43,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c7e70f5adf27c26564bc9bc160230a0, REOPEN/MOVE because future has completed 2024-11-11T07:30:43,312 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:30:43,312 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:30:43,334 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T07:30:43,465 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:43,465 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:30:43,465 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing 6c7e70f5adf27c26564bc9bc160230a0, disabling compactions & flushes 2024-11-11T07:30:43,465 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:43,465 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:43,465 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. after waiting 0 ms 2024-11-11T07:30:43,465 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:43,469 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T07:30:43,470 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:43,470 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for 6c7e70f5adf27c26564bc9bc160230a0: Waiting for close lock at 1731310243465Running coprocessor pre-close hooks at 1731310243465Disabling compacts and flushes for region at 1731310243465Disabling writes for close at 1731310243465Writing region close event to WAL at 1731310243466 (+1 ms)Running coprocessor post-close hooks at 1731310243470 (+4 ms)Closed at 1731310243470 2024-11-11T07:30:43,470 WARN [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: 6c7e70f5adf27c26564bc9bc160230a0 to self. 2024-11-11T07:30:43,472 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:43,472 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=6c7e70f5adf27c26564bc9bc160230a0, regionState=CLOSED 2024-11-11T07:30:43,474 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:30:43,476 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-11T07:30:43,476 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure 6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 in 163 msec 2024-11-11T07:30:43,477 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c7e70f5adf27c26564bc9bc160230a0, REOPEN/MOVE; state=CLOSED, location=ce2cf0244921,33345,1731310155369; forceNewPlan=false, retain=true 2024-11-11T07:30:43,628 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=6c7e70f5adf27c26564bc9bc160230a0, regionState=OPENING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:43,629 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c7e70f5adf27c26564bc9bc160230a0, REOPEN/MOVE because future has completed 2024-11-11T07:30:43,630 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:30:43,784 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:43,784 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:30:43,785 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:43,785 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:30:43,785 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:43,785 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:43,786 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:43,787 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:30:43,787 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c7e70f5adf27c26564bc9bc160230a0 columnFamilyName A 2024-11-11T07:30:43,788 DEBUG [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:43,789 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.HStore(327): Store=6c7e70f5adf27c26564bc9bc160230a0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:30:43,789 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:43,790 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:30:43,790 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c7e70f5adf27c26564bc9bc160230a0 columnFamilyName B 2024-11-11T07:30:43,790 DEBUG [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:43,790 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.HStore(327): Store=6c7e70f5adf27c26564bc9bc160230a0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:30:43,790 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:43,791 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:30:43,791 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6c7e70f5adf27c26564bc9bc160230a0 columnFamilyName C 2024-11-11T07:30:43,791 DEBUG [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:43,792 INFO [StoreOpener-6c7e70f5adf27c26564bc9bc160230a0-1 {}] regionserver.HStore(327): Store=6c7e70f5adf27c26564bc9bc160230a0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:30:43,792 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:43,792 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:43,792 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:43,793 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:43,794 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:43,794 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:43,794 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:30:43,796 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:43,796 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened 6c7e70f5adf27c26564bc9bc160230a0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68247839, jitterRate=0.016972050070762634}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:30:43,796 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:43,797 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for 6c7e70f5adf27c26564bc9bc160230a0: Running coprocessor pre-open hook at 1731310243785Writing region info on filesystem at 1731310243785Initializing all the Stores at 1731310243786 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310243786Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310243786Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310243786Cleaning up temporary data from old regions at 1731310243794 (+8 ms)Running coprocessor post-open hooks at 1731310243796 (+2 ms)Region opened successfully at 1731310243797 (+1 ms) 2024-11-11T07:30:43,798 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., pid=94, masterSystemTime=1731310243781 2024-11-11T07:30:43,800 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:43,800 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:43,800 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=6c7e70f5adf27c26564bc9bc160230a0, regionState=OPEN, openSeqNum=5, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:43,802 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:30:43,804 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-11T07:30:43,804 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure 6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 in 174 msec 2024-11-11T07:30:43,806 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-11T07:30:43,806 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c7e70f5adf27c26564bc9bc160230a0, REOPEN/MOVE in 495 msec 2024-11-11T07:30:43,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-11T07:30:43,808 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 499 msec 2024-11-11T07:30:43,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 922 msec 2024-11-11T07:30:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-11T07:30:43,811 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:43,812 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9ff38b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,812 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:43,812 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:43,813 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:43,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:43,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:43,813 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4882bb07, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:43,814 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:43,815 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:43,815 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@591d4020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,816 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62ec6412, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,816 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:43,816 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:43,817 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:43,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:43,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:43,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be0e814, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:43,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:43,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:43,818 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60640, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:43,819 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20847ee8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,820 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37f2378c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,820 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:43,820 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:43,821 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ec0e7da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:43,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:43,822 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60670, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:43,823 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30624d0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,823 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43cc49f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,823 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:43,823 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:43,824 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:43,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:43,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:43,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1481cfb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:43,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:43,825 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:43,825 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:43,826 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a8e76f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,827 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e0f0eef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,827 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:43,827 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:43,827 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:43,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:43,827 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b8f2c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:43,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:43,829 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60710, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:43,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45706d2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65616af9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,830 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:43,830 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:43,830 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:43,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:43,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:43,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30339dad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:43,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:43,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:43,832 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:43,832 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43af04e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,833 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74481d40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,833 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:43,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:43,834 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:43,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:43,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:43,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12dd58c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,834 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:43,835 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:43,835 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60748, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:43,836 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34cf52ea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,837 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@514b3d51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,837 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:43,837 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:43,838 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3983455c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:43,838 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:43,839 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:43,840 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5af1b015, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,841 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b14e3fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,841 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:43,841 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:43,842 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52cc992b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:43,842 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:43,843 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60794, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:43,844 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ebfe2fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,845 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75e264c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,845 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:30:43,845 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:30:43,846 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:30:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:30:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:30:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ab007d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,846 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:30:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:30:43,847 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:30:43,848 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:30:43,848 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28a841fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:30:43,849 DEBUG [Thread-1594 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:43,849 DEBUG [Thread-1598 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:43,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:43,850 DEBUG [Thread-1594 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:43,850 DEBUG [Thread-1594 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:43,850 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:43,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:43,851 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:30:43,854 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:30:43,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:43,862 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T07:30:43,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:43,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:43,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:43,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:43,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:43,862 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:43,863 DEBUG [Thread-1607 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:43,864 DEBUG [Thread-1611 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:43,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:43,865 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:43,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:43,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:43,866 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:43,866 DEBUG [Thread-1602 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:43,866 DEBUG [Thread-1613 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:43,867 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:43,867 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56496, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:43,867 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:43,868 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:43,869 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:43,870 DEBUG [Thread-1605 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:43,870 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:30:43,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:30:43,871 DEBUG [Thread-1602 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:43,872 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:30:43,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-11T07:30:43,874 DEBUG [Thread-1602 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:43,874 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T07:30:43,876 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:43,876 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:43,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:43,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:43,883 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:43,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:30:43,889 DEBUG [Thread-1609 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:43,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:43,890 DEBUG [Thread-1609 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:43,892 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56544, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:43,893 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:30:43,895 DEBUG [Thread-1602 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:30:43,901 DEBUG [Thread-1596 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:43,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:43,902 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:43,904 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:43,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:30:43,907 DEBUG [Thread-1600 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:30:43,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:30:43,907 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:30:43,909 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:30:43,910 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:30:43,954 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:43,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115d306f8893d944a9b7c1b8abd17a9c3a_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310243857/Put/seqid=0 2024-11-11T07:30:43,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731310243901, max timestamp=1731310243951Num uniques 859; 2024-11-11T07:30:43,958 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T07:30:43,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731310243901, max timestamp=1731310243951Num uniques 859; 2024-11-11T07:30:43,960 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T07:30:43,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731310243901, max timestamp=1731310243951Num uniques 859; 2024-11-11T07:30:43,961 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T07:30:43,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310303935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:43,963 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T07:30:44,029 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310304009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742177_1353 (size=12154) 2024-11-11T07:30:44,032 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:44,035 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310304026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,036 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411115d306f8893d944a9b7c1b8abd17a9c3a_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115d306f8893d944a9b7c1b8abd17a9c3a_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T07:30:44,039 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/f185e5e9cf7a4dc492ccdd9c4c0788d6, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:44,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/f185e5e9cf7a4dc492ccdd9c4c0788d6 is 175, key is test_row_0/A:col10/1731310243857/Put/seqid=0 2024-11-11T07:30:44,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:44,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:44,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:44,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:44,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:44,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:44,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1731310304027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310304029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742178_1354 (size=30955) 2024-11-11T07:30:44,056 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/f185e5e9cf7a4dc492ccdd9c4c0788d6 2024-11-11T07:30:44,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310304075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/185bb1db267b4a9c8f624867e958b599 is 50, key is test_row_0/B:col10/1731310243857/Put/seqid=0 2024-11-11T07:30:44,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742179_1355 (size=12001) 2024-11-11T07:30:44,100 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/185bb1db267b4a9c8f624867e958b599 2024-11-11T07:30:44,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/f2eafde55fb440fb93195b5fb29d9a17 is 50, key is test_row_0/C:col10/1731310243857/Put/seqid=0 2024-11-11T07:30:44,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742180_1356 (size=12001) 2024-11-11T07:30:44,147 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/f2eafde55fb440fb93195b5fb29d9a17 2024-11-11T07:30:44,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310304136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/f185e5e9cf7a4dc492ccdd9c4c0788d6 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/f185e5e9cf7a4dc492ccdd9c4c0788d6 2024-11-11T07:30:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,152 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310304145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,156 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,156 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/f185e5e9cf7a4dc492ccdd9c4c0788d6, entries=150, sequenceid=15, filesize=30.2 K 2024-11-11T07:30:44,157 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/185bb1db267b4a9c8f624867e958b599 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/185bb1db267b4a9c8f624867e958b599 2024-11-11T07:30:44,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/185bb1db267b4a9c8f624867e958b599, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T07:30:44,166 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/f2eafde55fb440fb93195b5fb29d9a17 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f2eafde55fb440fb93195b5fb29d9a17 2024-11-11T07:30:44,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f2eafde55fb440fb93195b5fb29d9a17, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T07:30:44,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 6c7e70f5adf27c26564bc9bc160230a0 in 311ms, sequenceid=15, compaction requested=false 2024-11-11T07:30:44,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:44,177 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-11T07:30:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:44,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T07:30:44,196 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T07:30:44,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:44,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:44,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:44,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:44,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:44,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:44,233 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b4e0abe6d80048808f25100cc82e1477_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310244027/Put/seqid=0 2024-11-11T07:30:44,270 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1731310304247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,271 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,271 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310304248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,271 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742181_1357 (size=17034) 2024-11-11T07:30:44,281 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:44,288 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b4e0abe6d80048808f25100cc82e1477_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b4e0abe6d80048808f25100cc82e1477_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:44,289 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/787df8e8f6c4472aade64eb76ea5d41d, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:44,290 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/787df8e8f6c4472aade64eb76ea5d41d is 175, key is test_row_0/A:col10/1731310244027/Put/seqid=0 2024-11-11T07:30:44,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310304284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742182_1358 (size=48139) 2024-11-11T07:30:44,324 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=50, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/787df8e8f6c4472aade64eb76ea5d41d 2024-11-11T07:30:44,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/bdbc76046b3441af8b6eb267d8f75699 is 50, key is test_row_0/B:col10/1731310244027/Put/seqid=0 2024-11-11T07:30:44,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T07:30:44,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:44,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:44,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:44,355 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:44,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:44,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:44,376 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310304354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,376 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,377 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310304364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,377 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742183_1359 (size=12001) 2024-11-11T07:30:44,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/bdbc76046b3441af8b6eb267d8f75699 2024-11-11T07:30:44,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310304375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1731310304375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,417 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/955f8c8847a64786b99b3691ce93b63d is 50, key is test_row_0/C:col10/1731310244027/Put/seqid=0 2024-11-11T07:30:44,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742184_1360 (size=12001) 2024-11-11T07:30:44,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/955f8c8847a64786b99b3691ce93b63d 2024-11-11T07:30:44,460 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/787df8e8f6c4472aade64eb76ea5d41d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/787df8e8f6c4472aade64eb76ea5d41d 2024-11-11T07:30:44,467 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/787df8e8f6c4472aade64eb76ea5d41d, entries=250, sequenceid=50, filesize=47.0 K 2024-11-11T07:30:44,467 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/bdbc76046b3441af8b6eb267d8f75699 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/bdbc76046b3441af8b6eb267d8f75699 2024-11-11T07:30:44,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/bdbc76046b3441af8b6eb267d8f75699, entries=150, sequenceid=50, filesize=11.7 K 2024-11-11T07:30:44,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/955f8c8847a64786b99b3691ce93b63d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/955f8c8847a64786b99b3691ce93b63d 2024-11-11T07:30:44,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/955f8c8847a64786b99b3691ce93b63d, entries=150, sequenceid=50, filesize=11.7 K 2024-11-11T07:30:44,480 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=46.96 KB/48090 for 6c7e70f5adf27c26564bc9bc160230a0 in 302ms, sequenceid=50, compaction requested=false 2024-11-11T07:30:44,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:44,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T07:30:44,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-11T07:30:44,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:44,511 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T07:30:44,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:44,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:44,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:44,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:44,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:44,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:44,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111929b132f4cc44a8fba2041ff25427380_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310244239/Put/seqid=0 2024-11-11T07:30:44,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742185_1361 (size=12154) 2024-11-11T07:30:44,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:44,587 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111929b132f4cc44a8fba2041ff25427380_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111929b132f4cc44a8fba2041ff25427380_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:44,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/251dd0d4fc7943d7bfaa2c9ff67598bf, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:44,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/251dd0d4fc7943d7bfaa2c9ff67598bf is 175, key is test_row_0/A:col10/1731310244239/Put/seqid=0 2024-11-11T07:30:44,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:44,625 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742186_1362 (size=30955) 2024-11-11T07:30:44,635 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=60, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/251dd0d4fc7943d7bfaa2c9ff67598bf 2024-11-11T07:30:44,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/e95471e35cd041dfb9a4131edbddd9e2 is 50, key is test_row_0/B:col10/1731310244239/Put/seqid=0 2024-11-11T07:30:44,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742187_1363 (size=12001) 2024-11-11T07:30:44,724 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/e95471e35cd041dfb9a4131edbddd9e2 2024-11-11T07:30:44,723 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310304697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,724 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1731310304696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,724 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,725 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,725 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731310244693, max timestamp=1731310244718Num uniques 859; 2024-11-11T07:30:44,725 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T07:30:44,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731310244693, max timestamp=1731310244718Num uniques 859; 2024-11-11T07:30:44,727 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T07:30:44,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731310244693, max timestamp=1731310244718Num uniques 859; 2024-11-11T07:30:44,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T07:30:44,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/46e35cd2f9554341bb6279f1b759d28c is 50, key is test_row_0/C:col10/1731310244239/Put/seqid=0 2024-11-11T07:30:44,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742188_1364 (size=12001) 2024-11-11T07:30:44,817 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=60 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/46e35cd2f9554341bb6279f1b759d28c 2024-11-11T07:30:44,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/251dd0d4fc7943d7bfaa2c9ff67598bf as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/251dd0d4fc7943d7bfaa2c9ff67598bf 2024-11-11T07:30:44,835 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/251dd0d4fc7943d7bfaa2c9ff67598bf, entries=150, sequenceid=60, filesize=30.2 K 2024-11-11T07:30:44,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/e95471e35cd041dfb9a4131edbddd9e2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/e95471e35cd041dfb9a4131edbddd9e2 2024-11-11T07:30:44,844 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/e95471e35cd041dfb9a4131edbddd9e2, entries=150, sequenceid=60, filesize=11.7 K 2024-11-11T07:30:44,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/46e35cd2f9554341bb6279f1b759d28c as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/46e35cd2f9554341bb6279f1b759d28c 2024-11-11T07:30:44,864 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/46e35cd2f9554341bb6279f1b759d28c, entries=150, sequenceid=60, filesize=11.7 K 2024-11-11T07:30:44,865 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=120.76 KB/123660 for 6c7e70f5adf27c26564bc9bc160230a0 in 355ms, sequenceid=60, compaction requested=true 2024-11-11T07:30:44,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:44,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:44,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-11T07:30:44,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-11T07:30:44,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-11T07:30:44,869 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-11T07:30:44,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 991 msec 2024-11-11T07:30:44,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:44,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:44,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:44,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:44,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:44,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:44,870 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:44,871 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.0030 sec 2024-11-11T07:30:44,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d9b67a8bb3774c4abc58e7f8d0f0918f_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310244861/Put/seqid=0 2024-11-11T07:30:44,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742189_1365 (size=12154) 2024-11-11T07:30:44,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310304899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,928 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1731310304901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310304902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,929 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310304904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,930 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:44,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310304906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:44,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-11T07:30:45,017 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:45,017 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:45,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-11T07:30:45,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T07:30:45,029 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:45,035 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:45,035 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:45,051 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310305036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1731310305037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,052 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,053 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310305038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310305038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310305038, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,059 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,059 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T07:30:45,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T07:30:45,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:45,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:45,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:45,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:45,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:45,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:45,272 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310305254, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,272 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310305265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,283 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1731310305265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310305266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310305266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,285 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,326 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:45,330 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d9b67a8bb3774c4abc58e7f8d0f0918f_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d9b67a8bb3774c4abc58e7f8d0f0918f_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:45,331 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/76a0b9a88a6a4e9c921d026abdaa34e2, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:45,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/76a0b9a88a6a4e9c921d026abdaa34e2 is 175, key is test_row_0/A:col10/1731310244861/Put/seqid=0 2024-11-11T07:30:45,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742190_1366 (size=30955) 2024-11-11T07:30:45,343 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/76a0b9a88a6a4e9c921d026abdaa34e2 2024-11-11T07:30:45,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T07:30:45,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T07:30:45,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:45,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:45,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:45,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:45,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:45,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:45,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/c5a27bc1a8924483b31f4172319a800c is 50, key is test_row_0/B:col10/1731310244861/Put/seqid=0 2024-11-11T07:30:45,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742191_1367 (size=12001) 2024-11-11T07:30:45,394 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/c5a27bc1a8924483b31f4172319a800c 2024-11-11T07:30:45,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/d3486eaa41764214933c8aeb0d243c68 is 50, key is test_row_0/C:col10/1731310244861/Put/seqid=0 2024-11-11T07:30:45,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742192_1368 (size=12001) 2024-11-11T07:30:45,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/d3486eaa41764214933c8aeb0d243c68 2024-11-11T07:30:45,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/76a0b9a88a6a4e9c921d026abdaa34e2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/76a0b9a88a6a4e9c921d026abdaa34e2 2024-11-11T07:30:45,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/76a0b9a88a6a4e9c921d026abdaa34e2, entries=150, sequenceid=95, filesize=30.2 K 2024-11-11T07:30:45,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/c5a27bc1a8924483b31f4172319a800c as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c5a27bc1a8924483b31f4172319a800c 2024-11-11T07:30:45,423 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c5a27bc1a8924483b31f4172319a800c, entries=150, sequenceid=95, filesize=11.7 K 2024-11-11T07:30:45,424 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/d3486eaa41764214933c8aeb0d243c68 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d3486eaa41764214933c8aeb0d243c68 2024-11-11T07:30:45,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d3486eaa41764214933c8aeb0d243c68, entries=150, sequenceid=95, filesize=11.7 K 2024-11-11T07:30:45,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=53.67 KB/54960 for 6c7e70f5adf27c26564bc9bc160230a0 in 560ms, sequenceid=95, compaction requested=true 2024-11-11T07:30:45,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:45,429 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:45,429 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:45,430 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:45,430 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:45,430 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/A is initiating minor compaction (all files) 2024-11-11T07:30:45,430 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/A in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:45,430 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/f185e5e9cf7a4dc492ccdd9c4c0788d6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/787df8e8f6c4472aade64eb76ea5d41d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/251dd0d4fc7943d7bfaa2c9ff67598bf, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/76a0b9a88a6a4e9c921d026abdaa34e2] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=137.7 K 2024-11-11T07:30:45,430 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:45,430 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/f185e5e9cf7a4dc492ccdd9c4c0788d6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/787df8e8f6c4472aade64eb76ea5d41d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/251dd0d4fc7943d7bfaa2c9ff67598bf, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/76a0b9a88a6a4e9c921d026abdaa34e2] 2024-11-11T07:30:45,431 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:45,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:45,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:45,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:45,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:45,431 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting f185e5e9cf7a4dc492ccdd9c4c0788d6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731310243857 2024-11-11T07:30:45,432 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 787df8e8f6c4472aade64eb76ea5d41d, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731310243980 2024-11-11T07:30:45,432 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 251dd0d4fc7943d7bfaa2c9ff67598bf, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731310244216 2024-11-11T07:30:45,432 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76a0b9a88a6a4e9c921d026abdaa34e2, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731310244823 2024-11-11T07:30:45,432 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:45,432 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/B is initiating minor compaction (all files) 2024-11-11T07:30:45,433 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/B in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:45,433 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/185bb1db267b4a9c8f624867e958b599, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/bdbc76046b3441af8b6eb267d8f75699, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/e95471e35cd041dfb9a4131edbddd9e2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c5a27bc1a8924483b31f4172319a800c] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=46.9 K 2024-11-11T07:30:45,433 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 185bb1db267b4a9c8f624867e958b599, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731310243857 2024-11-11T07:30:45,433 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting bdbc76046b3441af8b6eb267d8f75699, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731310244027 2024-11-11T07:30:45,434 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting e95471e35cd041dfb9a4131edbddd9e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731310244216 2024-11-11T07:30:45,434 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting c5a27bc1a8924483b31f4172319a800c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731310244823 2024-11-11T07:30:45,438 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:45,446 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#B#compaction#321 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:45,446 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/92bda31176a140f6b855544bce1915b9 is 50, key is test_row_0/B:col10/1731310244861/Put/seqid=0 2024-11-11T07:30:45,455 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111110a7569eb4d045dab9829e1d0c702ad7_6c7e70f5adf27c26564bc9bc160230a0 store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:45,458 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111110a7569eb4d045dab9829e1d0c702ad7_6c7e70f5adf27c26564bc9bc160230a0, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:45,458 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111110a7569eb4d045dab9829e1d0c702ad7_6c7e70f5adf27c26564bc9bc160230a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:45,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742194_1370 (size=4469) 2024-11-11T07:30:45,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742193_1369 (size=12139) 2024-11-11T07:30:45,501 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/92bda31176a140f6b855544bce1915b9 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/92bda31176a140f6b855544bce1915b9 2024-11-11T07:30:45,506 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/B of 6c7e70f5adf27c26564bc9bc160230a0 into 92bda31176a140f6b855544bce1915b9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:45,506 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:45,506 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/B, priority=12, startTime=1731310245430; duration=0sec 2024-11-11T07:30:45,506 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:45,506 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:B 2024-11-11T07:30:45,506 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:45,507 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:45,507 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/C is initiating minor compaction (all files) 2024-11-11T07:30:45,507 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/C in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:45,508 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f2eafde55fb440fb93195b5fb29d9a17, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/955f8c8847a64786b99b3691ce93b63d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/46e35cd2f9554341bb6279f1b759d28c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d3486eaa41764214933c8aeb0d243c68] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=46.9 K 2024-11-11T07:30:45,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-11T07:30:45,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:45,508 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T07:30:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:45,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:45,510 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f2eafde55fb440fb93195b5fb29d9a17, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731310243857 2024-11-11T07:30:45,511 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 955f8c8847a64786b99b3691ce93b63d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731310244027 2024-11-11T07:30:45,513 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 46e35cd2f9554341bb6279f1b759d28c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=60, earliestPutTs=1731310244216 2024-11-11T07:30:45,513 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting d3486eaa41764214933c8aeb0d243c68, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731310244823 2024-11-11T07:30:45,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111623f889444dd4fd69dab30f07a08e73e_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310244899/Put/seqid=0 2024-11-11T07:30:45,530 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#C#compaction#323 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:45,530 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/4fcd9448162148b9b113b1b93b794abb is 50, key is test_row_0/C:col10/1731310244861/Put/seqid=0 2024-11-11T07:30:45,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742195_1371 (size=12154) 2024-11-11T07:30:45,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:45,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:45,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:45,593 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111623f889444dd4fd69dab30f07a08e73e_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111623f889444dd4fd69dab30f07a08e73e_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:45,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/01699738f6c746919c58a86763a36712, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:45,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/01699738f6c746919c58a86763a36712 is 175, key is test_row_0/A:col10/1731310244899/Put/seqid=0 2024-11-11T07:30:45,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742196_1372 (size=12139) 2024-11-11T07:30:45,610 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/4fcd9448162148b9b113b1b93b794abb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/4fcd9448162148b9b113b1b93b794abb 2024-11-11T07:30:45,620 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/C of 6c7e70f5adf27c26564bc9bc160230a0 into 4fcd9448162148b9b113b1b93b794abb(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:45,620 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:45,620 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/C, priority=12, startTime=1731310245431; duration=0sec 2024-11-11T07:30:45,620 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:45,620 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:C 2024-11-11T07:30:45,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742197_1373 (size=30955) 2024-11-11T07:30:45,633 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=107, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/01699738f6c746919c58a86763a36712 2024-11-11T07:30:45,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/68acf95de2e84a2f913f4d4d0bbd99c5 is 50, key is test_row_0/B:col10/1731310244899/Put/seqid=0 2024-11-11T07:30:45,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T07:30:45,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742198_1374 (size=12001) 2024-11-11T07:30:45,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731310245616, max timestamp=1731310245656Num uniques 859; 2024-11-11T07:30:45,674 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T07:30:45,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310245650, max timestamp=1731310245656Num uniques 859; 2024-11-11T07:30:45,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:30:45,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310245650, max timestamp=1731310245656Num uniques 859; 2024-11-11T07:30:45,681 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:30:45,746 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310305728, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310305732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310305732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310305734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,753 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1731310305734, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,756 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,757 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,793 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T07:30:45,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310305856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310305856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310305856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,860 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1731310305865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:45,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310305865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:45,884 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:45,888 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#A#compaction#320 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:45,889 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/9becb4d43162463ab42b704162a4d30b is 175, key is test_row_0/A:col10/1731310244861/Put/seqid=0 2024-11-11T07:30:45,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742199_1375 (size=31093) 2024-11-11T07:30:45,936 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/9becb4d43162463ab42b704162a4d30b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/9becb4d43162463ab42b704162a4d30b 2024-11-11T07:30:45,944 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/A of 6c7e70f5adf27c26564bc9bc160230a0 into 9becb4d43162463ab42b704162a4d30b(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:45,944 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:45,944 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/A, priority=12, startTime=1731310245429; duration=0sec 2024-11-11T07:30:45,944 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:45,944 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:A 2024-11-11T07:30:46,066 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/68acf95de2e84a2f913f4d4d0bbd99c5 2024-11-11T07:30:46,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:46,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310306065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,078 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:46,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310306065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:46,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:46,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310306065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:46,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/66d6ca2be9604bca80b7892607447e14 is 50, key is test_row_0/C:col10/1731310244899/Put/seqid=0 2024-11-11T07:30:46,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742200_1376 (size=12001) 2024-11-11T07:30:46,105 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:46,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1731310306095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:46,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:46,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310306095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:46,106 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:46,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:46,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T07:30:46,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310306385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:46,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:46,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310306385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:46,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:46,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310306385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,390 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:46,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1731310306415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:46,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:46,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310306415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:46,428 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,429 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:46,505 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/66d6ca2be9604bca80b7892607447e14 2024-11-11T07:30:46,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/01699738f6c746919c58a86763a36712 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/01699738f6c746919c58a86763a36712 2024-11-11T07:30:46,523 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/01699738f6c746919c58a86763a36712, entries=150, sequenceid=107, filesize=30.2 K 2024-11-11T07:30:46,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/68acf95de2e84a2f913f4d4d0bbd99c5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/68acf95de2e84a2f913f4d4d0bbd99c5 2024-11-11T07:30:46,540 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/68acf95de2e84a2f913f4d4d0bbd99c5, entries=150, sequenceid=107, filesize=11.7 K 2024-11-11T07:30:46,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/66d6ca2be9604bca80b7892607447e14 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/66d6ca2be9604bca80b7892607447e14 2024-11-11T07:30:46,545 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/66d6ca2be9604bca80b7892607447e14, entries=150, sequenceid=107, filesize=11.7 K 2024-11-11T07:30:46,546 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=136.42 KB/139690 for 6c7e70f5adf27c26564bc9bc160230a0 in 1038ms, sequenceid=107, compaction requested=false 2024-11-11T07:30:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:46,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-11T07:30:46,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-11T07:30:46,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-11T07:30:46,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5130 sec 2024-11-11T07:30:46,552 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.5340 sec 2024-11-11T07:30:46,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:46,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=143.13 KB heapSize=405.30 KB 2024-11-11T07:30:46,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:46,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:46,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:46,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:46,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:46,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:46,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ae8b84e4efbd4f019aa7b086a90ef22d_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310245733/Put/seqid=0 2024-11-11T07:30:46,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742201_1377 (size=14794) 2024-11-11T07:30:46,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:46,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310306934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:46,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:46,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1731310306935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:46,957 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310306936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,957 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310306937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,958 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,959 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:46,961 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:46,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310306943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:46,962 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:47,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310307064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:47,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:47,079 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:47,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310307064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:47,080 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310307064, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,080 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:47,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-11T07:30:47,175 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:47,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-11T07:30:47,178 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:47,179 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:47,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:47,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T07:30:47,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T07:30:47,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310307286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:47,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,299 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:47,299 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:47,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310307286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:47,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:47,300 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:47,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310307286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,301 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:47,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T07:30:47,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:47,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:47,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:47,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:47,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:47,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:47,349 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:47,354 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ae8b84e4efbd4f019aa7b086a90ef22d_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ae8b84e4efbd4f019aa7b086a90ef22d_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:47,355 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/12e3aa9c7a68499a9dcb8115268a35bb, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:47,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/12e3aa9c7a68499a9dcb8115268a35bb is 175, key is test_row_0/A:col10/1731310245733/Put/seqid=0 2024-11-11T07:30:47,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742202_1378 (size=39749) 2024-11-11T07:30:47,372 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=146, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/12e3aa9c7a68499a9dcb8115268a35bb 2024-11-11T07:30:47,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/1ae36fdfd62d478b87494ebe758e38ce is 50, key is test_row_0/B:col10/1731310245733/Put/seqid=0 2024-11-11T07:30:47,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742203_1379 (size=12151) 2024-11-11T07:30:47,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/1ae36fdfd62d478b87494ebe758e38ce 2024-11-11T07:30:47,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/f29884bd96be41c29caa2411dc31ea59 is 50, key is test_row_0/C:col10/1731310245733/Put/seqid=0 2024-11-11T07:30:47,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742204_1380 (size=12151) 2024-11-11T07:30:47,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=146 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/f29884bd96be41c29caa2411dc31ea59 2024-11-11T07:30:47,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/12e3aa9c7a68499a9dcb8115268a35bb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/12e3aa9c7a68499a9dcb8115268a35bb 2024-11-11T07:30:47,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T07:30:47,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:47,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:47,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:47,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:47,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:47,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T07:30:47,509 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/12e3aa9c7a68499a9dcb8115268a35bb, entries=200, sequenceid=146, filesize=38.8 K 2024-11-11T07:30:47,510 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/1ae36fdfd62d478b87494ebe758e38ce as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/1ae36fdfd62d478b87494ebe758e38ce 2024-11-11T07:30:47,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/1ae36fdfd62d478b87494ebe758e38ce, entries=150, sequenceid=146, filesize=11.9 K 2024-11-11T07:30:47,514 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/f29884bd96be41c29caa2411dc31ea59 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f29884bd96be41c29caa2411dc31ea59 2024-11-11T07:30:47,525 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f29884bd96be41c29caa2411dc31ea59, entries=150, sequenceid=146, filesize=11.9 K 2024-11-11T07:30:47,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~143.13 KB/146560, heapSize ~405.25 KB/414976, currentSize=40.25 KB/41220 for 6c7e70f5adf27c26564bc9bc160230a0 in 614ms, sequenceid=146, compaction requested=true 2024-11-11T07:30:47,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:47,526 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:47,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:47,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:47,526 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:47,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:47,527 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101797 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:47,527 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/A is initiating minor compaction (all files) 2024-11-11T07:30:47,527 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/A in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:47,527 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/9becb4d43162463ab42b704162a4d30b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/01699738f6c746919c58a86763a36712, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/12e3aa9c7a68499a9dcb8115268a35bb] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=99.4 K 2024-11-11T07:30:47,527 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:47,528 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/9becb4d43162463ab42b704162a4d30b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/01699738f6c746919c58a86763a36712, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/12e3aa9c7a68499a9dcb8115268a35bb] 2024-11-11T07:30:47,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:47,528 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:47,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:47,528 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/B is initiating minor compaction (all files) 2024-11-11T07:30:47,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:47,528 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/B in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:47,528 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/92bda31176a140f6b855544bce1915b9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/68acf95de2e84a2f913f4d4d0bbd99c5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/1ae36fdfd62d478b87494ebe758e38ce] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=35.4 K 2024-11-11T07:30:47,528 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9becb4d43162463ab42b704162a4d30b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731310244823 2024-11-11T07:30:47,528 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 92bda31176a140f6b855544bce1915b9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731310244823 2024-11-11T07:30:47,529 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 01699738f6c746919c58a86763a36712, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731310244899 2024-11-11T07:30:47,529 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 68acf95de2e84a2f913f4d4d0bbd99c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731310244899 2024-11-11T07:30:47,529 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12e3aa9c7a68499a9dcb8115268a35bb, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731310245730 2024-11-11T07:30:47,529 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ae36fdfd62d478b87494ebe758e38ce, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731310245730 2024-11-11T07:30:47,545 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#B#compaction#329 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:47,545 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/072141a17a99486ea541eb7cd3006309 is 50, key is test_row_0/B:col10/1731310245733/Put/seqid=0 2024-11-11T07:30:47,551 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:47,578 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411110646be829b7f43b8a383c488b97be466_6c7e70f5adf27c26564bc9bc160230a0 store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:47,581 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411110646be829b7f43b8a383c488b97be466_6c7e70f5adf27c26564bc9bc160230a0, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:47,581 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110646be829b7f43b8a383c488b97be466_6c7e70f5adf27c26564bc9bc160230a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:47,594 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742205_1381 (size=12391) 2024-11-11T07:30:47,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742206_1382 (size=4469) 2024-11-11T07:30:47,603 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/072141a17a99486ea541eb7cd3006309 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/072141a17a99486ea541eb7cd3006309 2024-11-11T07:30:47,610 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/B of 6c7e70f5adf27c26564bc9bc160230a0 into 072141a17a99486ea541eb7cd3006309(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:47,610 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:47,610 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/B, priority=13, startTime=1731310247526; duration=0sec 2024-11-11T07:30:47,610 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:47,610 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:B 2024-11-11T07:30:47,610 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:47,612 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36291 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:47,612 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/C is initiating minor compaction (all files) 2024-11-11T07:30:47,612 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/C in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:47,612 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/4fcd9448162148b9b113b1b93b794abb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/66d6ca2be9604bca80b7892607447e14, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f29884bd96be41c29caa2411dc31ea59] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=35.4 K 2024-11-11T07:30:47,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:47,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T07:30:47,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:47,612 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:47,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:47,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:47,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:47,613 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:47,613 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 4fcd9448162148b9b113b1b93b794abb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731310244823 2024-11-11T07:30:47,613 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 66d6ca2be9604bca80b7892607447e14, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731310244899 2024-11-11T07:30:47,614 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f29884bd96be41c29caa2411dc31ea59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731310245730 2024-11-11T07:30:47,625 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b7b0df26a4d1484b817ff194d1948808_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310246915/Put/seqid=0 2024-11-11T07:30:47,643 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#C#compaction#332 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:47,643 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/208e24377553410f9aa28d67a4b1b6ae is 50, key is test_row_0/C:col10/1731310245733/Put/seqid=0 2024-11-11T07:30:47,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742207_1383 (size=14794) 2024-11-11T07:30:47,645 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:47,652 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b7b0df26a4d1484b817ff194d1948808_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b7b0df26a4d1484b817ff194d1948808_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:47,653 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/54706287047448919decb07c60f827f2, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:47,653 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/54706287047448919decb07c60f827f2 is 175, key is test_row_0/A:col10/1731310246915/Put/seqid=0 2024-11-11T07:30:47,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T07:30:47,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:47,656 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:47,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:47,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742208_1384 (size=12391) 2024-11-11T07:30:47,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742209_1385 (size=39749) 2024-11-11T07:30:47,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:47,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310307714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,730 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:47,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310307729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:47,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:47,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310307729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:47,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:47,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731310247688, max timestamp=1731310247728Num uniques 859; 2024-11-11T07:30:47,732 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T07:30:47,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731310247688, max timestamp=1731310247728Num uniques 859; 2024-11-11T07:30:47,733 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T07:30:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,733 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:47,739 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731310247688, max timestamp=1731310247728Num uniques 859; 2024-11-11T07:30:47,740 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T07:30:47,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T07:30:47,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:47,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:47,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:47,809 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:47,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:47,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:47,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T07:30:47,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:47,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310307930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,951 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:47,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T07:30:47,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:47,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:47,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:47,966 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:47,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:47,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310307950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:47,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:47,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310307950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:47,971 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:47,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310307965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:47,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:47,971 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_1' in 6c7e70f5adf27c26564bc9bc160230a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:47,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,972 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:47,991 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:47,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1731310307974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:47,994 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_0' in 6c7e70f5adf27c26564bc9bc160230a0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2259 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:47,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:47,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:47,999 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#A#compaction#330 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:48,000 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/58cb4f093b7e4dfb82d07ee6b984ce2b is 175, key is test_row_0/A:col10/1731310245733/Put/seqid=0 2024-11-11T07:30:48,010 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742210_1386 (size=31345) 2024-11-11T07:30:48,017 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/58cb4f093b7e4dfb82d07ee6b984ce2b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/58cb4f093b7e4dfb82d07ee6b984ce2b 2024-11-11T07:30:48,026 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/A of 6c7e70f5adf27c26564bc9bc160230a0 into 58cb4f093b7e4dfb82d07ee6b984ce2b(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:48,026 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:48,026 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/A, priority=13, startTime=1731310247526; duration=0sec 2024-11-11T07:30:48,026 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:48,026 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:A 2024-11-11T07:30:48,062 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:48,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310308056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,065 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:48,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:48,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310308079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,087 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:48,087 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:48,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310308079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,089 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:48,099 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/208e24377553410f9aa28d67a4b1b6ae as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/208e24377553410f9aa28d67a4b1b6ae 2024-11-11T07:30:48,104 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/C of 6c7e70f5adf27c26564bc9bc160230a0 into 208e24377553410f9aa28d67a4b1b6ae(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:48,104 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:48,104 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/C, priority=13, startTime=1731310247528; duration=0sec 2024-11-11T07:30:48,104 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:48,104 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:C 2024-11-11T07:30:48,114 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/54706287047448919decb07c60f827f2 2024-11-11T07:30:48,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T07:30:48,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:48,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:48,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:48,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:48,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:48,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:48,130 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/c70ec86a8d42402f80a8276be29adbf0 is 50, key is test_row_0/B:col10/1731310246915/Put/seqid=0 2024-11-11T07:30:48,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742211_1387 (size=12151) 2024-11-11T07:30:48,167 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/c70ec86a8d42402f80a8276be29adbf0 2024-11-11T07:30:48,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/7d2a0cc163bf4afdadb3be06f72041ed is 50, key is test_row_0/C:col10/1731310246915/Put/seqid=0 2024-11-11T07:30:48,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742212_1388 (size=12151) 2024-11-11T07:30:48,186 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/7d2a0cc163bf4afdadb3be06f72041ed 2024-11-11T07:30:48,194 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/54706287047448919decb07c60f827f2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/54706287047448919decb07c60f827f2 2024-11-11T07:30:48,198 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/54706287047448919decb07c60f827f2, entries=200, sequenceid=157, filesize=38.8 K 2024-11-11T07:30:48,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/c70ec86a8d42402f80a8276be29adbf0 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c70ec86a8d42402f80a8276be29adbf0 2024-11-11T07:30:48,203 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c70ec86a8d42402f80a8276be29adbf0, entries=150, sequenceid=157, filesize=11.9 K 2024-11-11T07:30:48,204 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/7d2a0cc163bf4afdadb3be06f72041ed as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/7d2a0cc163bf4afdadb3be06f72041ed 2024-11-11T07:30:48,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/7d2a0cc163bf4afdadb3be06f72041ed, entries=150, sequenceid=157, filesize=11.9 K 2024-11-11T07:30:48,216 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=132.99 KB/136185 for 6c7e70f5adf27c26564bc9bc160230a0 in 604ms, sequenceid=157, compaction requested=false 2024-11-11T07:30:48,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:48,272 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-11T07:30:48,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:48,273 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=132.99 KB heapSize=382.24 KB 2024-11-11T07:30:48,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:48,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:48,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:48,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:48,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:48,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:48,284 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:48,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:48,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a01b389728634abbbb709d22c80ddb0d_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310247927/Put/seqid=0 2024-11-11T07:30:48,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T07:30:48,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742213_1389 (size=12304) 2024-11-11T07:30:48,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:48,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310308337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,343 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:48,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:48,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310308339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:48,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,357 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:48,357 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:48,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310308341, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:48,360 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:48,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:48,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310308445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:48,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310308464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:48,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310308464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:48,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:48,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310308674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:48,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310308685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,705 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:48,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:48,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310308685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:48,706 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:48,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:48,736 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a01b389728634abbbb709d22c80ddb0d_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a01b389728634abbbb709d22c80ddb0d_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:48,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/e9fd86eefde34c97804dbb3a918ae00a, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:48,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/e9fd86eefde34c97804dbb3a918ae00a is 175, key is test_row_0/A:col10/1731310247927/Put/seqid=0 2024-11-11T07:30:48,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742214_1390 (size=31105) 2024-11-11T07:30:49,015 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:49,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310308999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:49,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:49,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:49,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:49,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310309015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:49,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:49,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310309015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:49,029 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:49,152 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=196, memsize=44.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/e9fd86eefde34c97804dbb3a918ae00a 2024-11-11T07:30:49,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/8f05f8c594f74d28b88025e794afeaba is 50, key is test_row_0/B:col10/1731310247927/Put/seqid=0 2024-11-11T07:30:49,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742215_1391 (size=12151) 2024-11-11T07:30:49,194 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/8f05f8c594f74d28b88025e794afeaba 2024-11-11T07:30:49,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/88262fdd99b147f4a335a3c47525c09f is 50, key is test_row_0/C:col10/1731310247927/Put/seqid=0 2024-11-11T07:30:49,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742216_1392 (size=12151) 2024-11-11T07:30:49,215 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.33 KB at sequenceid=196 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/88262fdd99b147f4a335a3c47525c09f 2024-11-11T07:30:49,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/e9fd86eefde34c97804dbb3a918ae00a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e9fd86eefde34c97804dbb3a918ae00a 2024-11-11T07:30:49,223 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e9fd86eefde34c97804dbb3a918ae00a, entries=150, sequenceid=196, filesize=30.4 K 2024-11-11T07:30:49,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/8f05f8c594f74d28b88025e794afeaba as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/8f05f8c594f74d28b88025e794afeaba 2024-11-11T07:30:49,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,232 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/8f05f8c594f74d28b88025e794afeaba, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T07:30:49,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/88262fdd99b147f4a335a3c47525c09f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/88262fdd99b147f4a335a3c47525c09f 2024-11-11T07:30:49,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,242 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/88262fdd99b147f4a335a3c47525c09f, entries=150, sequenceid=196, filesize=11.9 K 2024-11-11T07:30:49,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,243 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~132.99 KB/136185, heapSize ~382.20 KB/391368, currentSize=53.67 KB/54960 for 6c7e70f5adf27c26564bc9bc160230a0 in 970ms, sequenceid=196, compaction requested=true 2024-11-11T07:30:49,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:49,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:49,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-11T07:30:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-11T07:30:49,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-11T07:30:49,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0690 sec 2024-11-11T07:30:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,253 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.0760 sec 2024-11-11T07:30:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-11T07:30:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,335 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:49,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,339 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-11T07:30:49,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,341 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:49,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T07:30:49,341 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:49,342 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:49,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T07:30:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,494 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-11T07:30:49,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,494 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T07:30:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:49,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:49,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:49,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:49,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:49,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111195edbc4ffa1147c383674723044e8260_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_1/A:col10/1731310248337/Put/seqid=0 2024-11-11T07:30:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742217_1393 (size=9814) 2024-11-11T07:30:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,559 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111195edbc4ffa1147c383674723044e8260_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111195edbc4ffa1147c383674723044e8260_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:49,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/a08a80af5bf2433da183d69fbc2285c8, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/a08a80af5bf2433da183d69fbc2285c8 is 175, key is test_row_1/A:col10/1731310248337/Put/seqid=0 2024-11-11T07:30:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742218_1394 (size=22461) 2024-11-11T07:30:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,593 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/a08a80af5bf2433da183d69fbc2285c8 2024-11-11T07:30:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/d6be1f5ceeaf493583a9d7c2ccc61dd9 is 50, key is test_row_1/B:col10/1731310248337/Put/seqid=0 2024-11-11T07:30:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T07:30:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742219_1395 (size=9757) 2024-11-11T07:30:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:49,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310249839, max timestamp=1731310249902Num uniques 859; 2024-11-11T07:30:49,909 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:30:49,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310249839, max timestamp=1731310249902Num uniques 859; 2024-11-11T07:30:49,911 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:30:49,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310249839, max timestamp=1731310249902Num uniques 859; 2024-11-11T07:30:49,916 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:30:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T07:30:50,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310309990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310309991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,009 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,009 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,013 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310310007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,013 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310310010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,024 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1731310310016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,027 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_0' in 6c7e70f5adf27c26564bc9bc160230a0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4292 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,027 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,057 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/d6be1f5ceeaf493583a9d7c2ccc61dd9 2024-11-11T07:30:50,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/686058b920bf483f80017f6a6eec2d5b is 50, key is test_row_1/C:col10/1731310248337/Put/seqid=0 2024-11-11T07:30:50,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742220_1396 (size=9757) 2024-11-11T07:30:50,125 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/686058b920bf483f80017f6a6eec2d5b 2024-11-11T07:30:50,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/a08a80af5bf2433da183d69fbc2285c8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/a08a80af5bf2433da183d69fbc2285c8 2024-11-11T07:30:50,134 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/a08a80af5bf2433da183d69fbc2285c8, entries=100, sequenceid=207, filesize=21.9 K 2024-11-11T07:30:50,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/d6be1f5ceeaf493583a9d7c2ccc61dd9 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d6be1f5ceeaf493583a9d7c2ccc61dd9 2024-11-11T07:30:50,139 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d6be1f5ceeaf493583a9d7c2ccc61dd9, entries=100, sequenceid=207, filesize=9.5 K 2024-11-11T07:30:50,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/686058b920bf483f80017f6a6eec2d5b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/686058b920bf483f80017f6a6eec2d5b 2024-11-11T07:30:50,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310310114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310310114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,143 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,144 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,144 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/686058b920bf483f80017f6a6eec2d5b, entries=100, sequenceid=207, filesize=9.5 K 2024-11-11T07:30:50,145 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 6c7e70f5adf27c26564bc9bc160230a0 in 651ms, sequenceid=207, compaction requested=true 2024-11-11T07:30:50,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:50,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:50,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-11T07:30:50,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-11T07:30:50,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-11T07:30:50,149 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 805 msec 2024-11-11T07:30:50,151 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 810 msec 2024-11-11T07:30:50,158 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-11T07:30:50,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:50,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:50,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:50,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:50,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:50,158 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:50,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:50,181 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a3138de334504ed784496b5c82426da1_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310250151/Put/seqid=0 2024-11-11T07:30:50,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742221_1397 (size=14794) 2024-11-11T07:30:50,268 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310310244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310310250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,269 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,270 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310310345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310310354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,383 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310310375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,384 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310310375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-11T07:30:50,475 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:50,475 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:50,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-11T07:30:50,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T07:30:50,479 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:50,482 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:50,482 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:50,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T07:30:50,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310310594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310310594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,611 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,612 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,621 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:50,625 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a3138de334504ed784496b5c82426da1_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a3138de334504ed784496b5c82426da1_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:50,626 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/1336199e218b4c929511eeb73ad20a2d, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:50,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/1336199e218b4c929511eeb73ad20a2d is 175, key is test_row_0/A:col10/1731310250151/Put/seqid=0 2024-11-11T07:30:50,637 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:30:50,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:50,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:50,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:50,639 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:50,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:50,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:50,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742222_1398 (size=39749) 2024-11-11T07:30:50,648 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=244, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/1336199e218b4c929511eeb73ad20a2d 2024-11-11T07:30:50,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/c7650788f74c4d07a23db151daede211 is 50, key is test_row_0/B:col10/1731310250151/Put/seqid=0 2024-11-11T07:30:50,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742223_1399 (size=12151) 2024-11-11T07:30:50,661 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/c7650788f74c4d07a23db151daede211 2024-11-11T07:30:50,669 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/ae37277256f24c959aece33a8404491f is 50, key is test_row_0/C:col10/1731310250151/Put/seqid=0 2024-11-11T07:30:50,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310310664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,680 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742224_1400 (size=12151) 2024-11-11T07:30:50,687 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310310676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,687 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:30:50,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:50,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:50,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:50,791 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:50,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:50,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:50,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T07:30:50,919 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310310914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,919 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,939 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:50,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310310924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:50,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:50,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:30:50,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:50,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:50,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:50,952 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:50,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:50,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,084 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/ae37277256f24c959aece33a8404491f 2024-11-11T07:30:51,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/1336199e218b4c929511eeb73ad20a2d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1336199e218b4c929511eeb73ad20a2d 2024-11-11T07:30:51,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1336199e218b4c929511eeb73ad20a2d, entries=200, sequenceid=244, filesize=38.8 K 2024-11-11T07:30:51,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/c7650788f74c4d07a23db151daede211 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c7650788f74c4d07a23db151daede211 2024-11-11T07:30:51,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c7650788f74c4d07a23db151daede211, entries=150, sequenceid=244, filesize=11.9 K 2024-11-11T07:30:51,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/ae37277256f24c959aece33a8404491f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ae37277256f24c959aece33a8404491f 2024-11-11T07:30:51,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T07:30:51,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:30:51,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:51,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:51,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:51,107 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,109 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ae37277256f24c959aece33a8404491f, entries=150, sequenceid=244, filesize=11.9 K 2024-11-11T07:30:51,110 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 6c7e70f5adf27c26564bc9bc160230a0 in 953ms, sequenceid=244, compaction requested=true 2024-11-11T07:30:51,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:51,110 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T07:30:51,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:51,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:51,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:51,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:51,110 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T07:30:51,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:51,110 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:51,112 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 164409 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T07:30:51,112 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58601 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T07:30:51,113 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/A is initiating minor compaction (all files) 2024-11-11T07:30:51,113 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/B is initiating minor compaction (all files) 2024-11-11T07:30:51,113 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/B in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:51,113 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/A in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:51,113 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/072141a17a99486ea541eb7cd3006309, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c70ec86a8d42402f80a8276be29adbf0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/8f05f8c594f74d28b88025e794afeaba, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d6be1f5ceeaf493583a9d7c2ccc61dd9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c7650788f74c4d07a23db151daede211] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=57.2 K 2024-11-11T07:30:51,113 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/58cb4f093b7e4dfb82d07ee6b984ce2b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/54706287047448919decb07c60f827f2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e9fd86eefde34c97804dbb3a918ae00a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/a08a80af5bf2433da183d69fbc2285c8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1336199e218b4c929511eeb73ad20a2d] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=160.6 K 2024-11-11T07:30:51,113 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:51,113 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/58cb4f093b7e4dfb82d07ee6b984ce2b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/54706287047448919decb07c60f827f2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e9fd86eefde34c97804dbb3a918ae00a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/a08a80af5bf2433da183d69fbc2285c8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1336199e218b4c929511eeb73ad20a2d] 2024-11-11T07:30:51,113 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 072141a17a99486ea541eb7cd3006309, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731310245730 2024-11-11T07:30:51,113 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting c70ec86a8d42402f80a8276be29adbf0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731310246915 2024-11-11T07:30:51,113 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58cb4f093b7e4dfb82d07ee6b984ce2b, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731310245730 2024-11-11T07:30:51,114 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 8f05f8c594f74d28b88025e794afeaba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731310247927 2024-11-11T07:30:51,114 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54706287047448919decb07c60f827f2, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731310246915 2024-11-11T07:30:51,114 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting d6be1f5ceeaf493583a9d7c2ccc61dd9, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731310248337 2024-11-11T07:30:51,114 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting e9fd86eefde34c97804dbb3a918ae00a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731310247927 2024-11-11T07:30:51,114 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting c7650788f74c4d07a23db151daede211, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731310249988 2024-11-11T07:30:51,114 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting a08a80af5bf2433da183d69fbc2285c8, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731310248337 2024-11-11T07:30:51,115 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1336199e218b4c929511eeb73ad20a2d, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731310249988 2024-11-11T07:30:51,130 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#B#compaction#344 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:51,130 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/d58c81b2c1dd4fbf851822f476ca061c is 50, key is test_row_0/B:col10/1731310250151/Put/seqid=0 2024-11-11T07:30:51,139 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:51,141 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411110de87aa092fc48e4a9dde5e7d6b7acab_6c7e70f5adf27c26564bc9bc160230a0 store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:51,144 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411110de87aa092fc48e4a9dde5e7d6b7acab_6c7e70f5adf27c26564bc9bc160230a0, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:51,145 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110de87aa092fc48e4a9dde5e7d6b7acab_6c7e70f5adf27c26564bc9bc160230a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:51,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742225_1401 (size=12561) 2024-11-11T07:30:51,172 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/d58c81b2c1dd4fbf851822f476ca061c as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d58c81b2c1dd4fbf851822f476ca061c 2024-11-11T07:30:51,177 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/B of 6c7e70f5adf27c26564bc9bc160230a0 into d58c81b2c1dd4fbf851822f476ca061c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:51,177 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:51,177 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/B, priority=11, startTime=1731310251110; duration=0sec 2024-11-11T07:30:51,178 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:51,178 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:B 2024-11-11T07:30:51,178 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T07:30:51,183 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 58601 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T07:30:51,183 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/C is initiating minor compaction (all files) 2024-11-11T07:30:51,183 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/C in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:51,183 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/208e24377553410f9aa28d67a4b1b6ae, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/7d2a0cc163bf4afdadb3be06f72041ed, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/88262fdd99b147f4a335a3c47525c09f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/686058b920bf483f80017f6a6eec2d5b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ae37277256f24c959aece33a8404491f] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=57.2 K 2024-11-11T07:30:51,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742226_1402 (size=4469) 2024-11-11T07:30:51,184 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 208e24377553410f9aa28d67a4b1b6ae, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=146, earliestPutTs=1731310245730 2024-11-11T07:30:51,184 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d2a0cc163bf4afdadb3be06f72041ed, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731310246915 2024-11-11T07:30:51,184 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 88262fdd99b147f4a335a3c47525c09f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=196, earliestPutTs=1731310247927 2024-11-11T07:30:51,185 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 686058b920bf483f80017f6a6eec2d5b, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1731310248337 2024-11-11T07:30:51,188 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#A#compaction#345 average throughput is 0.50 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:51,188 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting ae37277256f24c959aece33a8404491f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731310249988 2024-11-11T07:30:51,189 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/b2cf42e07fc149a1b736714c9e2719c8 is 175, key is test_row_0/A:col10/1731310250151/Put/seqid=0 2024-11-11T07:30:51,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:51,211 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T07:30:51,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:51,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:51,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:51,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:51,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:51,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:51,226 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#C#compaction#346 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:51,226 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/98b71eade37e4705a69ae60bbed7b9e9 is 50, key is test_row_0/C:col10/1731310250151/Put/seqid=0 2024-11-11T07:30:51,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742227_1403 (size=31515) 2024-11-11T07:30:51,263 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:30:51,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:51,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:51,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:51,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742228_1404 (size=12561) 2024-11-11T07:30:51,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d5797afd66c74bd883dfddb25f2dbc50_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310250236/Put/seqid=0 2024-11-11T07:30:51,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742229_1405 (size=17334) 2024-11-11T07:30:51,321 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:51,325 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d5797afd66c74bd883dfddb25f2dbc50_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5797afd66c74bd883dfddb25f2dbc50_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:51,329 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/8d9991d938984d43bff72019c0f4ef80, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:51,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/8d9991d938984d43bff72019c0f4ef80 is 175, key is test_row_0/A:col10/1731310250236/Put/seqid=0 2024-11-11T07:30:51,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742230_1406 (size=48439) 2024-11-11T07:30:51,344 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/8d9991d938984d43bff72019c0f4ef80 2024-11-11T07:30:51,356 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/256b3ecb9f874c12abd3275201cf1e10 is 50, key is test_row_0/B:col10/1731310250236/Put/seqid=0 2024-11-11T07:30:51,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742231_1407 (size=12201) 2024-11-11T07:30:51,384 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/256b3ecb9f874c12abd3275201cf1e10 2024-11-11T07:30:51,399 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/d12280f48610440b9b93472b0cf02b70 is 50, key is test_row_0/C:col10/1731310250236/Put/seqid=0 2024-11-11T07:30:51,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:51,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310311396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:51,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,405 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:51,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:51,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310311397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:51,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,414 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:51,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:30:51,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:51,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:51,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742232_1408 (size=12201) 2024-11-11T07:30:51,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/d12280f48610440b9b93472b0cf02b70 2024-11-11T07:30:51,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/8d9991d938984d43bff72019c0f4ef80 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/8d9991d938984d43bff72019c0f4ef80 2024-11-11T07:30:51,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:51,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310311435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:51,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:51,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/8d9991d938984d43bff72019c0f4ef80, entries=250, sequenceid=257, filesize=47.3 K 2024-11-11T07:30:51,452 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/256b3ecb9f874c12abd3275201cf1e10 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/256b3ecb9f874c12abd3275201cf1e10 2024-11-11T07:30:51,457 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/256b3ecb9f874c12abd3275201cf1e10, entries=150, sequenceid=257, filesize=11.9 K 2024-11-11T07:30:51,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/d12280f48610440b9b93472b0cf02b70 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d12280f48610440b9b93472b0cf02b70 2024-11-11T07:30:51,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d12280f48610440b9b93472b0cf02b70, entries=150, sequenceid=257, filesize=11.9 K 2024-11-11T07:30:51,462 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6c7e70f5adf27c26564bc9bc160230a0 in 251ms, sequenceid=257, compaction requested=false 2024-11-11T07:30:51,462 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:51,473 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310251369, max timestamp=1731310251470Num uniques 859; 2024-11-11T07:30:51,474 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:30:51,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:51,474 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-11T07:30:51,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:51,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:51,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:51,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:51,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:51,475 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:30:51,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111106b07c29d35646e8b2361fe9f4ecd227_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310251470/Put/seqid=0 2024-11-11T07:30:51,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742233_1409 (size=14994) 2024-11-11T07:30:51,574 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:30:51,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:51,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:51,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:51,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310311595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:51,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310311595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:51,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T07:30:51,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:51,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310311596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,617 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:51,647 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/b2cf42e07fc149a1b736714c9e2719c8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/b2cf42e07fc149a1b736714c9e2719c8 2024-11-11T07:30:51,653 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/A of 6c7e70f5adf27c26564bc9bc160230a0 into b2cf42e07fc149a1b736714c9e2719c8(size=30.8 K), total size for store is 78.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:51,653 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:51,653 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/A, priority=11, startTime=1731310251110; duration=0sec 2024-11-11T07:30:51,653 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:51,653 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:A 2024-11-11T07:30:51,675 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/98b71eade37e4705a69ae60bbed7b9e9 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/98b71eade37e4705a69ae60bbed7b9e9 2024-11-11T07:30:51,680 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/C of 6c7e70f5adf27c26564bc9bc160230a0 into 98b71eade37e4705a69ae60bbed7b9e9(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:51,680 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:51,680 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/C, priority=11, startTime=1731310251110; duration=0sec 2024-11-11T07:30:51,680 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:51,680 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:C 2024-11-11T07:30:51,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310311717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,718 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:51,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:51,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310311717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,719 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:51,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:30:51,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:51,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:51,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:51,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310311725, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:51,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:51,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:30:51,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:51,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:51,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:51,888 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,888 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:51,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310311927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:51,931 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:51,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310311927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:51,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,934 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:51,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,934 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:51,937 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:51,942 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111106b07c29d35646e8b2361fe9f4ecd227_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111106b07c29d35646e8b2361fe9f4ecd227_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:51,943 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/3a5d5022ab414540b816d32a126c79a5, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:51,944 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/3a5d5022ab414540b816d32a126c79a5 is 175, key is test_row_0/A:col10/1731310251470/Put/seqid=0 2024-11-11T07:30:51,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742234_1410 (size=39949) 2024-11-11T07:30:51,951 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=278, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/3a5d5022ab414540b816d32a126c79a5 2024-11-11T07:30:51,952 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:51,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310311946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:51,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:51,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/518af87183264c3e891bcb9905f63fa4 is 50, key is test_row_0/B:col10/1731310251470/Put/seqid=0 2024-11-11T07:30:51,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742235_1411 (size=12301) 2024-11-11T07:30:51,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/518af87183264c3e891bcb9905f63fa4 2024-11-11T07:30:51,986 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/9031b144d14c43c6a9853b4788e53246 is 50, key is test_row_0/C:col10/1731310251470/Put/seqid=0 2024-11-11T07:30:51,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742236_1412 (size=12301) 2024-11-11T07:30:51,991 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=278 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/9031b144d14c43c6a9853b4788e53246 2024-11-11T07:30:52,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/3a5d5022ab414540b816d32a126c79a5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/3a5d5022ab414540b816d32a126c79a5 2024-11-11T07:30:52,006 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/3a5d5022ab414540b816d32a126c79a5, entries=200, sequenceid=278, filesize=39.0 K 2024-11-11T07:30:52,007 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/518af87183264c3e891bcb9905f63fa4 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/518af87183264c3e891bcb9905f63fa4 2024-11-11T07:30:52,015 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/518af87183264c3e891bcb9905f63fa4, entries=150, sequenceid=278, filesize=12.0 K 2024-11-11T07:30:52,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/9031b144d14c43c6a9853b4788e53246 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/9031b144d14c43c6a9853b4788e53246 2024-11-11T07:30:52,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/9031b144d14c43c6a9853b4788e53246, entries=150, sequenceid=278, filesize=12.0 K 2024-11-11T07:30:52,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:30:52,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:52,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:52,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:52,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:52,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:52,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:52,044 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 6c7e70f5adf27c26564bc9bc160230a0 in 570ms, sequenceid=278, compaction requested=true 2024-11-11T07:30:52,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:52,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:52,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:52,044 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:52,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:52,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:52,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:52,044 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:30:52,044 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:52,045 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119903 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:52,045 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/A is initiating minor compaction (all files) 2024-11-11T07:30:52,045 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/A in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:52,045 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/b2cf42e07fc149a1b736714c9e2719c8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/8d9991d938984d43bff72019c0f4ef80, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/3a5d5022ab414540b816d32a126c79a5] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=117.1 K 2024-11-11T07:30:52,045 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:52,045 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/b2cf42e07fc149a1b736714c9e2719c8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/8d9991d938984d43bff72019c0f4ef80, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/3a5d5022ab414540b816d32a126c79a5] 2024-11-11T07:30:52,045 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37063 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:52,045 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/B is initiating minor compaction (all files) 2024-11-11T07:30:52,046 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/B in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:52,046 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d58c81b2c1dd4fbf851822f476ca061c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/256b3ecb9f874c12abd3275201cf1e10, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/518af87183264c3e891bcb9905f63fa4] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=36.2 K 2024-11-11T07:30:52,046 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2cf42e07fc149a1b736714c9e2719c8, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731310249988 2024-11-11T07:30:52,046 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting d58c81b2c1dd4fbf851822f476ca061c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731310249988 2024-11-11T07:30:52,046 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d9991d938984d43bff72019c0f4ef80, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731310250224 2024-11-11T07:30:52,046 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 256b3ecb9f874c12abd3275201cf1e10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731310250236 2024-11-11T07:30:52,047 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a5d5022ab414540b816d32a126c79a5, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731310251369 2024-11-11T07:30:52,047 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 518af87183264c3e891bcb9905f63fa4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731310251369 2024-11-11T07:30:52,055 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:52,065 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#B#compaction#354 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:52,065 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/5b7349723fa8450cbc9e4be2fb40fc7b is 50, key is test_row_0/B:col10/1731310251470/Put/seqid=0 2024-11-11T07:30:52,077 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111e9eda2ded0404456ae5e0bb9dd93ef5e_6c7e70f5adf27c26564bc9bc160230a0 store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:52,080 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111e9eda2ded0404456ae5e0bb9dd93ef5e_6c7e70f5adf27c26564bc9bc160230a0, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:52,080 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e9eda2ded0404456ae5e0bb9dd93ef5e_6c7e70f5adf27c26564bc9bc160230a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:52,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742237_1413 (size=12813) 2024-11-11T07:30:52,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742238_1414 (size=4469) 2024-11-11T07:30:52,142 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#A#compaction#353 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:52,142 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/13589f1d1e3f4aa59877cdd03c83d3cc is 175, key is test_row_0/A:col10/1731310251470/Put/seqid=0 2024-11-11T07:30:52,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-11T07:30:52,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:52,194 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T07:30:52,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:52,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:52,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:52,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:52,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:52,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:52,225 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a84ec0a34def4611b5f1550e52848d6a_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310251593/Put/seqid=0 2024-11-11T07:30:52,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742239_1415 (size=31767) 2024-11-11T07:30:52,254 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:52,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742240_1416 (size=12454) 2024-11-11T07:30:52,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:52,276 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a84ec0a34def4611b5f1550e52848d6a_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a84ec0a34def4611b5f1550e52848d6a_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:52,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/1902c68ade564831ae3812d4b2de9395, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:52,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/1902c68ade564831ae3812d4b2de9395 is 175, key is test_row_0/A:col10/1731310251593/Put/seqid=0 2024-11-11T07:30:52,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742241_1417 (size=31255) 2024-11-11T07:30:52,292 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=296, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/1902c68ade564831ae3812d4b2de9395 2024-11-11T07:30:52,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/dbddb667df524245b8c6ddf905e01d33 is 50, key is test_row_0/B:col10/1731310251593/Put/seqid=0 2024-11-11T07:30:52,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310312308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,330 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:52,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742242_1418 (size=12301) 2024-11-11T07:30:52,341 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310312326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,341 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:52,343 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/dbddb667df524245b8c6ddf905e01d33 2024-11-11T07:30:52,344 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310312330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:52,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/8ec20ea326704c11b46ee4f3a9204fd6 is 50, key is test_row_0/C:col10/1731310251593/Put/seqid=0 2024-11-11T07:30:52,358 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742243_1419 (size=12301) 2024-11-11T07:30:52,359 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/8ec20ea326704c11b46ee4f3a9204fd6 2024-11-11T07:30:52,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/1902c68ade564831ae3812d4b2de9395 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1902c68ade564831ae3812d4b2de9395 2024-11-11T07:30:52,372 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1902c68ade564831ae3812d4b2de9395, entries=150, sequenceid=296, filesize=30.5 K 2024-11-11T07:30:52,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/dbddb667df524245b8c6ddf905e01d33 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/dbddb667df524245b8c6ddf905e01d33 2024-11-11T07:30:52,378 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/dbddb667df524245b8c6ddf905e01d33, entries=150, sequenceid=296, filesize=12.0 K 2024-11-11T07:30:52,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/8ec20ea326704c11b46ee4f3a9204fd6 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/8ec20ea326704c11b46ee4f3a9204fd6 2024-11-11T07:30:52,383 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/8ec20ea326704c11b46ee4f3a9204fd6, entries=150, sequenceid=296, filesize=12.0 K 2024-11-11T07:30:52,384 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 6c7e70f5adf27c26564bc9bc160230a0 in 190ms, sequenceid=296, compaction requested=true 2024-11-11T07:30:52,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:52,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:52,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-11T07:30:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-11T07:30:52,390 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-11T07:30:52,391 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9040 sec 2024-11-11T07:30:52,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.9160 sec 2024-11-11T07:30:52,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T07:30:52,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:52,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:52,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:52,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:52,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:52,446 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:52,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119b3bd02078624384adb7934c24d424cb_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310252324/Put/seqid=0 2024-11-11T07:30:52,521 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310312507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,522 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:52,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742244_1420 (size=14994) 2024-11-11T07:30:52,532 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/5b7349723fa8450cbc9e4be2fb40fc7b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/5b7349723fa8450cbc9e4be2fb40fc7b 2024-11-11T07:30:52,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:52,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310312511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:52,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310312516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:52,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310312522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,550 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:52,553 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/B of 6c7e70f5adf27c26564bc9bc160230a0 into 5b7349723fa8450cbc9e4be2fb40fc7b(size=12.5 K), total size for store is 24.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:52,553 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:52,553 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/B, priority=13, startTime=1731310252044; duration=0sec 2024-11-11T07:30:52,553 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:52,553 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:B 2024-11-11T07:30:52,553 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:52,555 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49364 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:52,555 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/C is initiating minor compaction (all files) 2024-11-11T07:30:52,555 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/C in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:52,555 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/98b71eade37e4705a69ae60bbed7b9e9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d12280f48610440b9b93472b0cf02b70, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/9031b144d14c43c6a9853b4788e53246, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/8ec20ea326704c11b46ee4f3a9204fd6] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=48.2 K 2024-11-11T07:30:52,559 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 98b71eade37e4705a69ae60bbed7b9e9, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731310249988 2024-11-11T07:30:52,560 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting d12280f48610440b9b93472b0cf02b70, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731310250236 2024-11-11T07:30:52,560 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 9031b144d14c43c6a9853b4788e53246, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731310251369 2024-11-11T07:30:52,560 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ec20ea326704c11b46ee4f3a9204fd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731310251573 2024-11-11T07:30:52,577 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#C#compaction#359 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:52,578 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/02e859b1cbcb4a499c773fab91b076e1 is 50, key is test_row_0/C:col10/1731310251593/Put/seqid=0 2024-11-11T07:30:52,618 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742245_1421 (size=12847) 2024-11-11T07:30:52,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-11T07:30:52,625 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:52,625 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-11T07:30:52,627 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:52,629 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:52,629 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:52,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T07:30:52,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:52,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310312624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,642 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:52,651 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/13589f1d1e3f4aa59877cdd03c83d3cc as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/13589f1d1e3f4aa59877cdd03c83d3cc 2024-11-11T07:30:52,664 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/A of 6c7e70f5adf27c26564bc9bc160230a0 into 13589f1d1e3f4aa59877cdd03c83d3cc(size=31.0 K), total size for store is 61.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:52,664 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:52,664 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/A, priority=13, startTime=1731310252044; duration=0sec 2024-11-11T07:30:52,665 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:52,665 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:A 2024-11-11T07:30:52,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310312645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:52,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:52,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310312645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:52,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:52,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310312654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:52,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:52,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T07:30:52,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:30:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:52,799 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:52,799 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:52,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:52,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:52,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310312854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:52,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:52,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310312885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:52,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:52,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310312885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:52,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:52,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310312885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:52,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,894 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:52,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,894 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:52,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:52,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:52,921 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:52,925 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119b3bd02078624384adb7934c24d424cb_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119b3bd02078624384adb7934c24d424cb_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:52,930 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/301e552ddd7f44ed9da3a9d6919fdfdb, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:52,931 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/301e552ddd7f44ed9da3a9d6919fdfdb is 175, key is test_row_0/A:col10/1731310252324/Put/seqid=0 2024-11-11T07:30:52,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T07:30:52,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742246_1422 (size=39949) 2024-11-11T07:30:52,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:30:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:52,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:52,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:52,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:52,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:52,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,024 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/02e859b1cbcb4a499c773fab91b076e1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/02e859b1cbcb4a499c773fab91b076e1 2024-11-11T07:30:53,029 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/C of 6c7e70f5adf27c26564bc9bc160230a0 into 02e859b1cbcb4a499c773fab91b076e1(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:53,029 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:53,029 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/C, priority=12, startTime=1731310252044; duration=0sec 2024-11-11T07:30:53,030 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:53,030 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:C 2024-11-11T07:30:53,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:30:53,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:53,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:53,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:53,114 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:53,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310313174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:53,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:53,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:53,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:53,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310313205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:53,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:53,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310313205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:53,216 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:53,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:53,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310313205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:53,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:53,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T07:30:53,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:30:53,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:53,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:53,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:53,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,359 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/301e552ddd7f44ed9da3a9d6919fdfdb 2024-11-11T07:30:53,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/23f06fa4ae9d459da59c51002f793efa is 50, key is test_row_0/B:col10/1731310252324/Put/seqid=0 2024-11-11T07:30:53,427 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:30:53,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742247_1423 (size=12301) 2024-11-11T07:30:53,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:53,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:53,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:53,431 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,586 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:30:53,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:53,586 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:53,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:53,587 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310313685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:53,694 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:53,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310313729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:53,744 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:53,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310313729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:53,745 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310313730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:53,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:53,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:53,745 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:53,746 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:53,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:30:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:53,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:53,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:53,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T07:30:53,831 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/23f06fa4ae9d459da59c51002f793efa 2024-11-11T07:30:53,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/4228dbcafef642659fae83ece981d378 is 50, key is test_row_0/C:col10/1731310252324/Put/seqid=0 2024-11-11T07:30:53,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742248_1424 (size=12301) 2024-11-11T07:30:53,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:30:53,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:53,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:53,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:53,921 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:53,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:54,066 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:54,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1731310314058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:54,066 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_0' in 6c7e70f5adf27c26564bc9bc160230a0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8332 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:30:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,067 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:54,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:30:54,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:54,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:54,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:54,083 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:54,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:54,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:30:54,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:54,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:54,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:54,243 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:54,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:54,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:54,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/4228dbcafef642659fae83ece981d378 2024-11-11T07:30:54,308 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/301e552ddd7f44ed9da3a9d6919fdfdb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/301e552ddd7f44ed9da3a9d6919fdfdb 2024-11-11T07:30:54,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/301e552ddd7f44ed9da3a9d6919fdfdb, entries=200, sequenceid=313, filesize=39.0 K 2024-11-11T07:30:54,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/23f06fa4ae9d459da59c51002f793efa as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/23f06fa4ae9d459da59c51002f793efa 2024-11-11T07:30:54,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/23f06fa4ae9d459da59c51002f793efa, entries=150, sequenceid=313, filesize=12.0 K 2024-11-11T07:30:54,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/4228dbcafef642659fae83ece981d378 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/4228dbcafef642659fae83ece981d378 2024-11-11T07:30:54,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/4228dbcafef642659fae83ece981d378, entries=150, sequenceid=313, filesize=12.0 K 2024-11-11T07:30:54,337 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 6c7e70f5adf27c26564bc9bc160230a0 in 1891ms, sequenceid=313, compaction requested=true 2024-11-11T07:30:54,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:54,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:54,337 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:54,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:54,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:54,337 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:54,337 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:54,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:54,338 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:54,338 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:54,338 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102971 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:54,338 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/B is initiating minor compaction (all files) 2024-11-11T07:30:54,338 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/A is initiating minor compaction (all files) 2024-11-11T07:30:54,338 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/A in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:54,338 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/B in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:54,338 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/13589f1d1e3f4aa59877cdd03c83d3cc, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1902c68ade564831ae3812d4b2de9395, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/301e552ddd7f44ed9da3a9d6919fdfdb] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=100.6 K 2024-11-11T07:30:54,338 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/5b7349723fa8450cbc9e4be2fb40fc7b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/dbddb667df524245b8c6ddf905e01d33, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/23f06fa4ae9d459da59c51002f793efa] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=36.5 K 2024-11-11T07:30:54,338 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:54,338 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/13589f1d1e3f4aa59877cdd03c83d3cc, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1902c68ade564831ae3812d4b2de9395, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/301e552ddd7f44ed9da3a9d6919fdfdb] 2024-11-11T07:30:54,339 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b7349723fa8450cbc9e4be2fb40fc7b, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731310251369 2024-11-11T07:30:54,339 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13589f1d1e3f4aa59877cdd03c83d3cc, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=278, earliestPutTs=1731310251369 2024-11-11T07:30:54,339 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting dbddb667df524245b8c6ddf905e01d33, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731310251573 2024-11-11T07:30:54,339 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1902c68ade564831ae3812d4b2de9395, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731310251573 2024-11-11T07:30:54,339 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 23f06fa4ae9d459da59c51002f793efa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731310252305 2024-11-11T07:30:54,340 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 301e552ddd7f44ed9da3a9d6919fdfdb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731310252305 2024-11-11T07:30:54,359 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:54,369 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#B#compaction#363 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:54,369 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/b66bb6a854fe4579a04f161d7eb1739a is 50, key is test_row_0/B:col10/1731310252324/Put/seqid=0 2024-11-11T07:30:54,387 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411114b5cf6ab537047258aeeb9fe59f21138_6c7e70f5adf27c26564bc9bc160230a0 store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:54,389 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411114b5cf6ab537047258aeeb9fe59f21138_6c7e70f5adf27c26564bc9bc160230a0, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:54,389 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114b5cf6ab537047258aeeb9fe59f21138_6c7e70f5adf27c26564bc9bc160230a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:54,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-11T07:30:54,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:54,411 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T07:30:54,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:54,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:54,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:54,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:54,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:54,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:54,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742249_1425 (size=12915) 2024-11-11T07:30:54,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742250_1426 (size=4469) 2024-11-11T07:30:54,463 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#A#compaction#362 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:54,464 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/0d9a92eccfcd4cc9b86d958a8b3451ce is 175, key is test_row_0/A:col10/1731310252324/Put/seqid=0 2024-11-11T07:30:54,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114b6ebb3981e542fe86c3c0d81d3922bf_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310252476/Put/seqid=0 2024-11-11T07:30:54,528 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742251_1427 (size=31869) 2024-11-11T07:30:54,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742252_1428 (size=12454) 2024-11-11T07:30:54,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:54,554 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114b6ebb3981e542fe86c3c0d81d3922bf_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114b6ebb3981e542fe86c3c0d81d3922bf_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:54,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/5b3ccae13ee845fea3a7c295b40fc8d1, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:54,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/5b3ccae13ee845fea3a7c295b40fc8d1 is 175, key is test_row_0/A:col10/1731310252476/Put/seqid=0 2024-11-11T07:30:54,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742253_1429 (size=31255) 2024-11-11T07:30:54,720 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T07:30:54,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310314805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:54,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310314805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:54,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310314805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:54,826 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:54,826 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310314807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,827 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,828 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:54,845 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/b66bb6a854fe4579a04f161d7eb1739a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/b66bb6a854fe4579a04f161d7eb1739a 2024-11-11T07:30:54,854 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/B of 6c7e70f5adf27c26564bc9bc160230a0 into b66bb6a854fe4579a04f161d7eb1739a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:54,854 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:54,854 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/B, priority=13, startTime=1731310254337; duration=0sec 2024-11-11T07:30:54,855 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:54,855 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:B 2024-11-11T07:30:54,855 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T07:30:54,855 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T07:30:54,856 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T07:30:54,856 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. because compaction request was cancelled 2024-11-11T07:30:54,856 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:C 2024-11-11T07:30:54,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310314935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:54,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310314935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:54,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:54,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310314935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:54,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310314935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,944 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:54,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:54,945 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:54,946 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/0d9a92eccfcd4cc9b86d958a8b3451ce as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/0d9a92eccfcd4cc9b86d958a8b3451ce 2024-11-11T07:30:54,951 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/A of 6c7e70f5adf27c26564bc9bc160230a0 into 0d9a92eccfcd4cc9b86d958a8b3451ce(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:54,951 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:54,951 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/A, priority=13, startTime=1731310254337; duration=0sec 2024-11-11T07:30:54,952 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:54,952 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:A 2024-11-11T07:30:54,994 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=331, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/5b3ccae13ee845fea3a7c295b40fc8d1 2024-11-11T07:30:55,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/9c53f8eaf9584b7ba4a2d9e6416f6729 is 50, key is test_row_0/B:col10/1731310252476/Put/seqid=0 2024-11-11T07:30:55,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742254_1430 (size=12301) 2024-11-11T07:30:55,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:55,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310315145, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:55,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:55,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310315154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:55,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:55,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310315154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:55,161 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:55,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310315154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:55,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:55,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:55,419 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/9c53f8eaf9584b7ba4a2d9e6416f6729 2024-11-11T07:30:55,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/41a3e9b99bf84f14a67649a4c0958c81 is 50, key is test_row_0/C:col10/1731310252476/Put/seqid=0 2024-11-11T07:30:55,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742255_1431 (size=12301) 2024-11-11T07:30:55,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:55,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310315464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:55,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310315464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:55,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:55,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310315464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:55,480 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310315466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:55,480 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:55,859 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=331 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/41a3e9b99bf84f14a67649a4c0958c81 2024-11-11T07:30:55,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/5b3ccae13ee845fea3a7c295b40fc8d1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/5b3ccae13ee845fea3a7c295b40fc8d1 2024-11-11T07:30:55,889 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/5b3ccae13ee845fea3a7c295b40fc8d1, entries=150, sequenceid=331, filesize=30.5 K 2024-11-11T07:30:55,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/9c53f8eaf9584b7ba4a2d9e6416f6729 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/9c53f8eaf9584b7ba4a2d9e6416f6729 2024-11-11T07:30:55,902 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/9c53f8eaf9584b7ba4a2d9e6416f6729, entries=150, sequenceid=331, filesize=12.0 K 2024-11-11T07:30:55,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/41a3e9b99bf84f14a67649a4c0958c81 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/41a3e9b99bf84f14a67649a4c0958c81 2024-11-11T07:30:55,907 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/41a3e9b99bf84f14a67649a4c0958c81, entries=150, sequenceid=331, filesize=12.0 K 2024-11-11T07:30:55,908 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6c7e70f5adf27c26564bc9bc160230a0 in 1497ms, sequenceid=331, compaction requested=true 2024-11-11T07:30:55,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:55,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:55,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-11T07:30:55,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-11T07:30:55,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-11T07:30:55,912 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2810 sec 2024-11-11T07:30:55,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 3.2950 sec 2024-11-11T07:30:55,987 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T07:30:55,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:55,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:55,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:55,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:55,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:55,988 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:55,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:55,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111885d917171054354a42bd39375c1f620_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310255986/Put/seqid=0 2024-11-11T07:30:56,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742256_1432 (size=14994) 2024-11-11T07:30:56,002 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,006 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111885d917171054354a42bd39375c1f620_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111885d917171054354a42bd39375c1f620_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:56,007 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/940e59b19620410cbbaaae0f1da8dd9f, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:56,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/940e59b19620410cbbaaae0f1da8dd9f is 175, key is test_row_0/A:col10/1731310255986/Put/seqid=0 2024-11-11T07:30:56,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742257_1433 (size=39949) 2024-11-11T07:30:56,051 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:56,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310316033, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310316034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,052 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:56,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:56,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310316036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:56,055 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=351, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/940e59b19620410cbbaaae0f1da8dd9f 2024-11-11T07:30:56,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:56,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310316051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,073 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:56,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/d724e267965b4c8e8b8016cfeb408581 is 50, key is test_row_0/B:col10/1731310255986/Put/seqid=0 2024-11-11T07:30:56,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742258_1434 (size=12301) 2024-11-11T07:30:56,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/d724e267965b4c8e8b8016cfeb408581 2024-11-11T07:30:56,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310316155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:56,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310316156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,162 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,164 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:56,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:56,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310316164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:56,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,168 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:56,176 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/139a0aaee16c4792958593123aeebff3 is 50, key is test_row_0/C:col10/1731310255986/Put/seqid=0 2024-11-11T07:30:56,191 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310316175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:56,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,192 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:56,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742259_1435 (size=12301) 2024-11-11T07:30:56,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:56,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310316365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:56,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,372 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:56,386 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:56,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310316375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:56,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:56,387 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310316375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:56,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,387 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:56,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310316404, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,416 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,417 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:56,616 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=351 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/139a0aaee16c4792958593123aeebff3 2024-11-11T07:30:56,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/940e59b19620410cbbaaae0f1da8dd9f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/940e59b19620410cbbaaae0f1da8dd9f 2024-11-11T07:30:56,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/940e59b19620410cbbaaae0f1da8dd9f, entries=200, sequenceid=351, filesize=39.0 K 2024-11-11T07:30:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/d724e267965b4c8e8b8016cfeb408581 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d724e267965b4c8e8b8016cfeb408581 2024-11-11T07:30:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d724e267965b4c8e8b8016cfeb408581, entries=150, sequenceid=351, filesize=12.0 K 2024-11-11T07:30:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/139a0aaee16c4792958593123aeebff3 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/139a0aaee16c4792958593123aeebff3 2024-11-11T07:30:56,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/139a0aaee16c4792958593123aeebff3, entries=150, sequenceid=351, filesize=12.0 K 2024-11-11T07:30:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6c7e70f5adf27c26564bc9bc160230a0 in 657ms, sequenceid=351, compaction requested=true 2024-11-11T07:30:56,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,644 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:56,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:56,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:56,644 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:56,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:56,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:56,644 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:56,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,645 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:56,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,646 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:56,646 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103073 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:56,646 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/B is initiating minor compaction (all files) 2024-11-11T07:30:56,646 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/A is initiating minor compaction (all files) 2024-11-11T07:30:56,646 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/B in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:56,646 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/A in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:56,646 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/0d9a92eccfcd4cc9b86d958a8b3451ce, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/5b3ccae13ee845fea3a7c295b40fc8d1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/940e59b19620410cbbaaae0f1da8dd9f] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=100.7 K 2024-11-11T07:30:56,646 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/b66bb6a854fe4579a04f161d7eb1739a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/9c53f8eaf9584b7ba4a2d9e6416f6729, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d724e267965b4c8e8b8016cfeb408581] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=36.6 K 2024-11-11T07:30:56,646 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:56,646 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/0d9a92eccfcd4cc9b86d958a8b3451ce, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/5b3ccae13ee845fea3a7c295b40fc8d1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/940e59b19620410cbbaaae0f1da8dd9f] 2024-11-11T07:30:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,647 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting b66bb6a854fe4579a04f161d7eb1739a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731310252305 2024-11-11T07:30:56,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,647 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d9a92eccfcd4cc9b86d958a8b3451ce, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731310252305 2024-11-11T07:30:56,647 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 9c53f8eaf9584b7ba4a2d9e6416f6729, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731310252476 2024-11-11T07:30:56,647 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b3ccae13ee845fea3a7c295b40fc8d1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731310252476 2024-11-11T07:30:56,647 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting d724e267965b4c8e8b8016cfeb408581, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731310254783 2024-11-11T07:30:56,648 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 940e59b19620410cbbaaae0f1da8dd9f, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731310254783 2024-11-11T07:30:56,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,656 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,675 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#B#compaction#371 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:56,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,675 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/16c7e7dab1d14d75845c86c557b803ec is 50, key is test_row_0/B:col10/1731310255986/Put/seqid=0 2024-11-11T07:30:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,679 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111146ecddac4f2547978a98559521b5cee8_6c7e70f5adf27c26564bc9bc160230a0 store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,681 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111146ecddac4f2547978a98559521b5cee8_6c7e70f5adf27c26564bc9bc160230a0, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:56,681 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111146ecddac4f2547978a98559521b5cee8_6c7e70f5adf27c26564bc9bc160230a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:56,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T07:30:56,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:56,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:56,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:56,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:56,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:56,735 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742260_1436 (size=13017) 2024-11-11T07:30:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,751 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/16c7e7dab1d14d75845c86c557b803ec as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/16c7e7dab1d14d75845c86c557b803ec 2024-11-11T07:30:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742261_1437 (size=4469) 2024-11-11T07:30:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,756 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#A#compaction#370 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,757 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/a41e6ccf90074d768c592d8bc52fe698 is 175, key is test_row_0/A:col10/1731310255986/Put/seqid=0 2024-11-11T07:30:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,760 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/B of 6c7e70f5adf27c26564bc9bc160230a0 into 16c7e7dab1d14d75845c86c557b803ec(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:56,760 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:56,760 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/B, priority=13, startTime=1731310256644; duration=0sec 2024-11-11T07:30:56,760 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:56,760 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:B 2024-11-11T07:30:56,760 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:30:56,762 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49750 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:30:56,762 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/C is initiating minor compaction (all files) 2024-11-11T07:30:56,762 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/C in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,762 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/02e859b1cbcb4a499c773fab91b076e1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/4228dbcafef642659fae83ece981d378, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/41a3e9b99bf84f14a67649a4c0958c81, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/139a0aaee16c4792958593123aeebff3] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=48.6 K 2024-11-11T07:30:56,762 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 02e859b1cbcb4a499c773fab91b076e1, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1731310251573 2024-11-11T07:30:56,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,762 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 4228dbcafef642659fae83ece981d378, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731310252305 2024-11-11T07:30:56,763 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 41a3e9b99bf84f14a67649a4c0958c81, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=331, earliestPutTs=1731310252476 2024-11-11T07:30:56,763 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 139a0aaee16c4792958593123aeebff3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731310254783 2024-11-11T07:30:56,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,782 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d804a3855d06471eb4401245aa13a570_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_1/A:col10/1731310256733/Put/seqid=0 2024-11-11T07:30:56,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,789 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#C#compaction#373 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:56,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,790 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/ad6b0bb528b74b429702cd6d804bda4f is 50, key is test_row_0/C:col10/1731310255986/Put/seqid=0 2024-11-11T07:30:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-11T07:30:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,807 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:56,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-11T07:30:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,813 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:56,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T07:30:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,816 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:56,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:56,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742262_1438 (size=31971) 2024-11-11T07:30:56,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742263_1439 (size=12454) 2024-11-11T07:30:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,838 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,843 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d804a3855d06471eb4401245aa13a570_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d804a3855d06471eb4401245aa13a570_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:56,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,843 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/a41e6ccf90074d768c592d8bc52fe698 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/a41e6ccf90074d768c592d8bc52fe698 2024-11-11T07:30:56,844 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/e417d9b9d73f4a059ca946b0e7baabe1, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:56,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,844 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/e417d9b9d73f4a059ca946b0e7baabe1 is 175, key is test_row_1/A:col10/1731310256733/Put/seqid=0 2024-11-11T07:30:56,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,850 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/A of 6c7e70f5adf27c26564bc9bc160230a0 into a41e6ccf90074d768c592d8bc52fe698(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:56,850 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:56,850 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/A, priority=13, startTime=1731310256644; duration=0sec 2024-11-11T07:30:56,850 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:56,850 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:A 2024-11-11T07:30:56,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:56,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742264_1440 (size=12983) 2024-11-11T07:30:56,876 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/ad6b0bb528b74b429702cd6d804bda4f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ad6b0bb528b74b429702cd6d804bda4f 2024-11-11T07:30:56,881 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/C of 6c7e70f5adf27c26564bc9bc160230a0 into ad6b0bb528b74b429702cd6d804bda4f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:56,881 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:56,881 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/C, priority=12, startTime=1731310256644; duration=0sec 2024-11-11T07:30:56,881 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:56,881 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:C 2024-11-11T07:30:56,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742265_1441 (size=31251) 2024-11-11T07:30:56,915 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=366, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/e417d9b9d73f4a059ca946b0e7baabe1 2024-11-11T07:30:56,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T07:30:56,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/66209a605835431696034626333d9b72 is 50, key is test_row_1/B:col10/1731310256733/Put/seqid=0 2024-11-11T07:30:56,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:56,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310316906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:56,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:56,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310316908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,939 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,940 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:56,945 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:56,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310316920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,946 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:56,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310316923, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:56,946 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:56,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742266_1442 (size=9857) 2024-11-11T07:30:56,967 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/66209a605835431696034626333d9b72 2024-11-11T07:30:56,975 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:30:56,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:56,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:56,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:56,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:56,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:56,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:56,984 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/d1d0e5cb6ba84b15bca6abeacc620141 is 50, key is test_row_1/C:col10/1731310256733/Put/seqid=0 2024-11-11T07:30:57,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742267_1443 (size=9857) 2024-11-11T07:30:57,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:57,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310317047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:57,060 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:57,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310317047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,060 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:57,073 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:57,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310317059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:57,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,074 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:57,074 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:57,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310317059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:57,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:57,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:30:57,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:57,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:57,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:57,135 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:57,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T07:30:57,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:57,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:57,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310317265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:57,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310317265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,270 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:57,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310317285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:57,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:57,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310317285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:57,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,291 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:57,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,291 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:57,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:30:57,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:57,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:57,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:57,295 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:57,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:57,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:57,428 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/d1d0e5cb6ba84b15bca6abeacc620141 2024-11-11T07:30:57,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/e417d9b9d73f4a059ca946b0e7baabe1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e417d9b9d73f4a059ca946b0e7baabe1 2024-11-11T07:30:57,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T07:30:57,446 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e417d9b9d73f4a059ca946b0e7baabe1, entries=150, sequenceid=366, filesize=30.5 K 2024-11-11T07:30:57,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/66209a605835431696034626333d9b72 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/66209a605835431696034626333d9b72 2024-11-11T07:30:57,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:30:57,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:57,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:57,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:57,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:57,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:57,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:57,460 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/66209a605835431696034626333d9b72, entries=100, sequenceid=366, filesize=9.6 K 2024-11-11T07:30:57,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/d1d0e5cb6ba84b15bca6abeacc620141 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d1d0e5cb6ba84b15bca6abeacc620141 2024-11-11T07:30:57,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d1d0e5cb6ba84b15bca6abeacc620141, entries=100, sequenceid=366, filesize=9.6 K 2024-11-11T07:30:57,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6c7e70f5adf27c26564bc9bc160230a0 in 731ms, sequenceid=366, compaction requested=false 2024-11-11T07:30:57,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:57,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T07:30:57,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:57,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:57,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:57,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:57,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:57,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:57,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e99b6376e3b44c8480fc46b85407d164_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310256920/Put/seqid=0 2024-11-11T07:30:57,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742268_1444 (size=14994) 2024-11-11T07:30:57,600 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:57,604 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e99b6376e3b44c8480fc46b85407d164_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e99b6376e3b44c8480fc46b85407d164_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:57,605 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/d6f40a8b633b46d6bfe3e5c8496813c1, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:57,605 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/d6f40a8b633b46d6bfe3e5c8496813c1 is 175, key is test_row_0/A:col10/1731310256920/Put/seqid=0 2024-11-11T07:30:57,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:30:57,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:57,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:57,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:57,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:57,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:57,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742269_1445 (size=39949) 2024-11-11T07:30:57,620 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=387, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/d6f40a8b633b46d6bfe3e5c8496813c1 2024-11-11T07:30:57,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/1eb97a54c18f4dcb999b536a3f217b8a is 50, key is test_row_0/B:col10/1731310256920/Put/seqid=0 2024-11-11T07:30:57,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:57,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310317627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310317627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:57,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:57,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310317638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:57,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742270_1446 (size=12301) 2024-11-11T07:30:57,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/1eb97a54c18f4dcb999b536a3f217b8a 2024-11-11T07:30:57,656 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:57,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310317645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:57,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,658 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:57,664 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/5ef0fd1a8feb43058a358d416a1e509f is 50, key is test_row_0/C:col10/1731310256920/Put/seqid=0 2024-11-11T07:30:57,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742271_1447 (size=12301) 2024-11-11T07:30:57,670 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=387 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/5ef0fd1a8feb43058a358d416a1e509f 2024-11-11T07:30:57,677 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/d6f40a8b633b46d6bfe3e5c8496813c1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/d6f40a8b633b46d6bfe3e5c8496813c1 2024-11-11T07:30:57,681 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/d6f40a8b633b46d6bfe3e5c8496813c1, entries=200, sequenceid=387, filesize=39.0 K 2024-11-11T07:30:57,682 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/1eb97a54c18f4dcb999b536a3f217b8a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/1eb97a54c18f4dcb999b536a3f217b8a 2024-11-11T07:30:57,686 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/1eb97a54c18f4dcb999b536a3f217b8a, entries=150, sequenceid=387, filesize=12.0 K 2024-11-11T07:30:57,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/5ef0fd1a8feb43058a358d416a1e509f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/5ef0fd1a8feb43058a358d416a1e509f 2024-11-11T07:30:57,692 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/5ef0fd1a8feb43058a358d416a1e509f, entries=150, sequenceid=387, filesize=12.0 K 2024-11-11T07:30:57,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6c7e70f5adf27c26564bc9bc160230a0 in 123ms, sequenceid=387, compaction requested=true 2024-11-11T07:30:57,703 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:57,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:57,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:57,703 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:57,703 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:57,704 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35175 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:57,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:57,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:57,704 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/B is initiating minor compaction (all files) 2024-11-11T07:30:57,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:57,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:57,704 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/B in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:57,704 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/16c7e7dab1d14d75845c86c557b803ec, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/66209a605835431696034626333d9b72, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/1eb97a54c18f4dcb999b536a3f217b8a] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=34.4 K 2024-11-11T07:30:57,704 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103171 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:57,704 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/A is initiating minor compaction (all files) 2024-11-11T07:30:57,705 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/A in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:57,705 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 16c7e7dab1d14d75845c86c557b803ec, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731310254783 2024-11-11T07:30:57,705 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/a41e6ccf90074d768c592d8bc52fe698, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e417d9b9d73f4a059ca946b0e7baabe1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/d6f40a8b633b46d6bfe3e5c8496813c1] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=100.8 K 2024-11-11T07:30:57,705 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:57,705 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/a41e6ccf90074d768c592d8bc52fe698, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e417d9b9d73f4a059ca946b0e7baabe1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/d6f40a8b633b46d6bfe3e5c8496813c1] 2024-11-11T07:30:57,705 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 66209a605835431696034626333d9b72, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731310256048 2024-11-11T07:30:57,705 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 1eb97a54c18f4dcb999b536a3f217b8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731310256905 2024-11-11T07:30:57,705 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting a41e6ccf90074d768c592d8bc52fe698, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731310254783 2024-11-11T07:30:57,706 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting e417d9b9d73f4a059ca946b0e7baabe1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731310256034 2024-11-11T07:30:57,706 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6f40a8b633b46d6bfe3e5c8496813c1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731310256905 2024-11-11T07:30:57,717 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#B#compaction#379 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:57,717 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/37e846bc2d054579a07ebe9231c0d9d4 is 50, key is test_row_0/B:col10/1731310256920/Put/seqid=0 2024-11-11T07:30:57,732 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:57,751 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411118351ecb9186e4db5a11a0e862f7f0e9b_6c7e70f5adf27c26564bc9bc160230a0 store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:57,753 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411118351ecb9186e4db5a11a0e862f7f0e9b_6c7e70f5adf27c26564bc9bc160230a0, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:57,753 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118351ecb9186e4db5a11a0e862f7f0e9b_6c7e70f5adf27c26564bc9bc160230a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:57,760 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-11T07:30:57,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:57,760 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T07:30:57,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:57,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:57,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:57,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:57,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:57,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:57,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:57,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:57,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742273_1449 (size=4469) 2024-11-11T07:30:57,772 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#A#compaction#380 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:57,772 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/617eb9bcdc684ae290970659785dff91 is 175, key is test_row_0/A:col10/1731310256920/Put/seqid=0 2024-11-11T07:30:57,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118bac7ef083ed4c8babbefea027495841_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310257643/Put/seqid=0 2024-11-11T07:30:57,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742272_1448 (size=13119) 2024-11-11T07:30:57,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742274_1450 (size=32073) 2024-11-11T07:30:57,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742275_1451 (size=12454) 2024-11-11T07:30:57,864 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/617eb9bcdc684ae290970659785dff91 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/617eb9bcdc684ae290970659785dff91 2024-11-11T07:30:57,882 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/A of 6c7e70f5adf27c26564bc9bc160230a0 into 617eb9bcdc684ae290970659785dff91(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:57,882 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:57,882 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/A, priority=13, startTime=1731310257703; duration=0sec 2024-11-11T07:30:57,882 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:57,882 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:A 2024-11-11T07:30:57,883 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:57,884 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:57,884 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/C is initiating minor compaction (all files) 2024-11-11T07:30:57,884 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/C in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:57,884 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ad6b0bb528b74b429702cd6d804bda4f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d1d0e5cb6ba84b15bca6abeacc620141, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/5ef0fd1a8feb43058a358d416a1e509f] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=34.3 K 2024-11-11T07:30:57,884 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad6b0bb528b74b429702cd6d804bda4f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=351, earliestPutTs=1731310254783 2024-11-11T07:30:57,884 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1d0e5cb6ba84b15bca6abeacc620141, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731310256048 2024-11-11T07:30:57,886 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ef0fd1a8feb43058a358d416a1e509f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731310256905 2024-11-11T07:30:57,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310317860, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,892 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:57,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310317882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:57,912 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:57,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310317886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:57,913 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:57,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310317892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:57,914 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#C#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:57,914 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:57,915 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/ad81013a041a468ba867d12c0f349859 is 50, key is test_row_0/C:col10/1731310256920/Put/seqid=0 2024-11-11T07:30:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:57,916 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:57,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742276_1452 (size=13085) 2024-11-11T07:30:57,954 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/ad81013a041a468ba867d12c0f349859 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ad81013a041a468ba867d12c0f349859 2024-11-11T07:30:57,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T07:30:57,959 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/C of 6c7e70f5adf27c26564bc9bc160230a0 into ad81013a041a468ba867d12c0f349859(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:57,959 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:57,959 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/C, priority=13, startTime=1731310257704; duration=0sec 2024-11-11T07:30:57,960 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:57,960 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:C 2024-11-11T07:30:58,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310317994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,003 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310318025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,037 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,037 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310318025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,038 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310318025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,038 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,196 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/37e846bc2d054579a07ebe9231c0d9d4 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/37e846bc2d054579a07ebe9231c0d9d4 2024-11-11T07:30:58,200 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/B of 6c7e70f5adf27c26564bc9bc160230a0 into 37e846bc2d054579a07ebe9231c0d9d4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:58,201 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:58,201 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/B, priority=13, startTime=1731310257703; duration=0sec 2024-11-11T07:30:58,201 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:58,201 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:B 2024-11-11T07:30:58,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310318205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,212 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310318244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310318244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,253 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310318245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,255 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:58,268 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118bac7ef083ed4c8babbefea027495841_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118bac7ef083ed4c8babbefea027495841_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:58,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/e3541e505178446b8132476d874f3e0e, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:58,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/e3541e505178446b8132476d874f3e0e is 175, key is test_row_0/A:col10/1731310257643/Put/seqid=0 2024-11-11T07:30:58,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742277_1453 (size=31255) 2024-11-11T07:30:58,321 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=401, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/e3541e505178446b8132476d874f3e0e 2024-11-11T07:30:58,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/89afc6368deb4f9e8bf4aae26f050630 is 50, key is test_row_0/B:col10/1731310257643/Put/seqid=0 2024-11-11T07:30:58,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742278_1454 (size=12301) 2024-11-11T07:30:58,388 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/89afc6368deb4f9e8bf4aae26f050630 2024-11-11T07:30:58,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/24eea72a9a6340a78365cd4e309530f4 is 50, key is test_row_0/C:col10/1731310257643/Put/seqid=0 2024-11-11T07:30:58,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742279_1455 (size=12301) 2024-11-11T07:30:58,413 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/24eea72a9a6340a78365cd4e309530f4 2024-11-11T07:30:58,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/e3541e505178446b8132476d874f3e0e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e3541e505178446b8132476d874f3e0e 2024-11-11T07:30:58,425 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e3541e505178446b8132476d874f3e0e, entries=150, sequenceid=401, filesize=30.5 K 2024-11-11T07:30:58,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/89afc6368deb4f9e8bf4aae26f050630 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/89afc6368deb4f9e8bf4aae26f050630 2024-11-11T07:30:58,430 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/89afc6368deb4f9e8bf4aae26f050630, entries=150, sequenceid=401, filesize=12.0 K 2024-11-11T07:30:58,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/24eea72a9a6340a78365cd4e309530f4 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/24eea72a9a6340a78365cd4e309530f4 2024-11-11T07:30:58,434 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/24eea72a9a6340a78365cd4e309530f4, entries=150, sequenceid=401, filesize=12.0 K 2024-11-11T07:30:58,444 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 6c7e70f5adf27c26564bc9bc160230a0 in 684ms, sequenceid=401, compaction requested=false 2024-11-11T07:30:58,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:58,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:58,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-11T07:30:58,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-11T07:30:58,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-11T07:30:58,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6290 sec 2024-11-11T07:30:58,449 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.6370 sec 2024-11-11T07:30:58,520 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T07:30:58,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:58,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:58,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:58,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:58,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:58,520 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:58,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:58,549 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b982a20e73c04b96b7b6ab0d71f2d6ad_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310257885/Put/seqid=0 2024-11-11T07:30:58,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742280_1456 (size=14994) 2024-11-11T07:30:58,580 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:58,583 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b982a20e73c04b96b7b6ab0d71f2d6ad_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b982a20e73c04b96b7b6ab0d71f2d6ad_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:58,584 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/148f0628487743bca49fbe846c4b27d3, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:58,585 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/148f0628487743bca49fbe846c4b27d3 is 175, key is test_row_0/A:col10/1731310257885/Put/seqid=0 2024-11-11T07:30:58,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742281_1457 (size=39949) 2024-11-11T07:30:58,601 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=423, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/148f0628487743bca49fbe846c4b27d3 2024-11-11T07:30:58,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310318587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310318591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,610 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,611 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/13277e4a91c94132bf85c1f8c88cdad4 is 50, key is test_row_0/B:col10/1731310257885/Put/seqid=0 2024-11-11T07:30:58,633 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310318607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310318608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,634 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,635 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742282_1458 (size=12301) 2024-11-11T07:30:58,726 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310318714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,727 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310318715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,730 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310318745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310318745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,764 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310318934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310318935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,947 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-11T07:30:58,967 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:30:58,970 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:30:58,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-11T07:30:58,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T07:30:58,977 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:30:58,978 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:30:58,978 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:30:58,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310318975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,984 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,984 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:58,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310318976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:58,985 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:59,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/13277e4a91c94132bf85c1f8c88cdad4 2024-11-11T07:30:59,083 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/331119a96eec4d8a99b6528df92d5b14 is 50, key is test_row_0/C:col10/1731310257885/Put/seqid=0 2024-11-11T07:30:59,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T07:30:59,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742283_1459 (size=12301) 2024-11-11T07:30:59,124 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/331119a96eec4d8a99b6528df92d5b14 2024-11-11T07:30:59,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/148f0628487743bca49fbe846c4b27d3 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/148f0628487743bca49fbe846c4b27d3 2024-11-11T07:30:59,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/148f0628487743bca49fbe846c4b27d3, entries=200, sequenceid=423, filesize=39.0 K 2024-11-11T07:30:59,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/13277e4a91c94132bf85c1f8c88cdad4 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/13277e4a91c94132bf85c1f8c88cdad4 2024-11-11T07:30:59,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T07:30:59,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:59,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:59,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:59,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:59,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:59,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:59,135 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/13277e4a91c94132bf85c1f8c88cdad4, entries=150, sequenceid=423, filesize=12.0 K 2024-11-11T07:30:59,136 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/331119a96eec4d8a99b6528df92d5b14 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/331119a96eec4d8a99b6528df92d5b14 2024-11-11T07:30:59,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/331119a96eec4d8a99b6528df92d5b14, entries=150, sequenceid=423, filesize=12.0 K 2024-11-11T07:30:59,148 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 6c7e70f5adf27c26564bc9bc160230a0 in 627ms, sequenceid=423, compaction requested=true 2024-11-11T07:30:59,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:59,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:30:59,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:59,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:30:59,148 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:59,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:59,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:30:59,148 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:30:59,148 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:59,149 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103277 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:59,149 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:59,149 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/B is initiating minor compaction (all files) 2024-11-11T07:30:59,149 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/A is initiating minor compaction (all files) 2024-11-11T07:30:59,149 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/B in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:59,149 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/A in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:59,149 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/37e846bc2d054579a07ebe9231c0d9d4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/89afc6368deb4f9e8bf4aae26f050630, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/13277e4a91c94132bf85c1f8c88cdad4] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=36.8 K 2024-11-11T07:30:59,149 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/617eb9bcdc684ae290970659785dff91, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e3541e505178446b8132476d874f3e0e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/148f0628487743bca49fbe846c4b27d3] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=100.9 K 2024-11-11T07:30:59,149 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:59,149 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/617eb9bcdc684ae290970659785dff91, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e3541e505178446b8132476d874f3e0e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/148f0628487743bca49fbe846c4b27d3] 2024-11-11T07:30:59,149 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 37e846bc2d054579a07ebe9231c0d9d4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731310256905 2024-11-11T07:30:59,150 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 617eb9bcdc684ae290970659785dff91, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731310256905 2024-11-11T07:30:59,150 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89afc6368deb4f9e8bf4aae26f050630, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731310257624 2024-11-11T07:30:59,150 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 13277e4a91c94132bf85c1f8c88cdad4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731310257803 2024-11-11T07:30:59,150 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting e3541e505178446b8132476d874f3e0e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731310257624 2024-11-11T07:30:59,150 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 148f0628487743bca49fbe846c4b27d3, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731310257803 2024-11-11T07:30:59,165 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#B#compaction#388 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:59,165 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/11f4c2ecc27f4573871723fd978e0156 is 50, key is test_row_0/B:col10/1731310257885/Put/seqid=0 2024-11-11T07:30:59,175 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:59,207 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111527a91ddf918416596cbe8dd3131f8cf_6c7e70f5adf27c26564bc9bc160230a0 store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:59,209 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111527a91ddf918416596cbe8dd3131f8cf_6c7e70f5adf27c26564bc9bc160230a0, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:59,209 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111527a91ddf918416596cbe8dd3131f8cf_6c7e70f5adf27c26564bc9bc160230a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:59,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742284_1460 (size=13221) 2024-11-11T07:30:59,242 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/11f4c2ecc27f4573871723fd978e0156 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/11f4c2ecc27f4573871723fd978e0156 2024-11-11T07:30:59,248 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/B of 6c7e70f5adf27c26564bc9bc160230a0 into 11f4c2ecc27f4573871723fd978e0156(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:59,248 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:59,248 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/B, priority=13, startTime=1731310259148; duration=0sec 2024-11-11T07:30:59,248 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:30:59,248 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:B 2024-11-11T07:30:59,248 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:30:59,249 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:30:59,249 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/C is initiating minor compaction (all files) 2024-11-11T07:30:59,249 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/C in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:59,249 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ad81013a041a468ba867d12c0f349859, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/24eea72a9a6340a78365cd4e309530f4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/331119a96eec4d8a99b6528df92d5b14] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=36.8 K 2024-11-11T07:30:59,251 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad81013a041a468ba867d12c0f349859, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=387, earliestPutTs=1731310256905 2024-11-11T07:30:59,251 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24eea72a9a6340a78365cd4e309530f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731310257624 2024-11-11T07:30:59,251 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 331119a96eec4d8a99b6528df92d5b14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731310257803 2024-11-11T07:30:59,261 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#C#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:59,261 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/cbdd7d8cda5a41c3a39cbfe2f810cfa8 is 50, key is test_row_0/C:col10/1731310257885/Put/seqid=0 2024-11-11T07:30:59,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742285_1461 (size=4469) 2024-11-11T07:30:59,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T07:30:59,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:59,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:59,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:59,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:59,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:59,269 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:59,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:59,287 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T07:30:59,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:59,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:59,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:59,288 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:59,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:59,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:59,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T07:30:59,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111922077b91cb74e5d9bbed5ae93ae5e4f_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310258580/Put/seqid=0 2024-11-11T07:30:59,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742286_1462 (size=13187) 2024-11-11T07:30:59,328 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/cbdd7d8cda5a41c3a39cbfe2f810cfa8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/cbdd7d8cda5a41c3a39cbfe2f810cfa8 2024-11-11T07:30:59,333 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/C of 6c7e70f5adf27c26564bc9bc160230a0 into cbdd7d8cda5a41c3a39cbfe2f810cfa8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:30:59,333 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:59,334 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/C, priority=13, startTime=1731310259148; duration=0sec 2024-11-11T07:30:59,334 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:30:59,334 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:C 2024-11-11T07:30:59,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742287_1463 (size=17534) 2024-11-11T07:30:59,338 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:59,343 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111922077b91cb74e5d9bbed5ae93ae5e4f_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111922077b91cb74e5d9bbed5ae93ae5e4f_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:59,344 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/145d67a6e51f463cba7ee3e63174c5d2, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:59,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/145d67a6e51f463cba7ee3e63174c5d2 is 175, key is test_row_0/A:col10/1731310258580/Put/seqid=0 2024-11-11T07:30:59,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742288_1464 (size=48639) 2024-11-11T07:30:59,376 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=439, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/145d67a6e51f463cba7ee3e63174c5d2 2024-11-11T07:30:59,377 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:59,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310319359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:59,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310319359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:59,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310319359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:59,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:59,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310319361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:59,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:59,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,381 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:59,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,381 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:59,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/7e751ffdd58241908977f210cf4846b6 is 50, key is test_row_0/B:col10/1731310258580/Put/seqid=0 2024-11-11T07:30:59,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742289_1465 (size=12301) 2024-11-11T07:30:59,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T07:30:59,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:59,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:59,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:59,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:59,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:59,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:30:59,446 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/7e751ffdd58241908977f210cf4846b6 2024-11-11T07:30:59,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/dc1847fcae25409fad97132f001eaaf1 is 50, key is test_row_0/C:col10/1731310258580/Put/seqid=0 2024-11-11T07:30:59,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742290_1466 (size=12301) 2024-11-11T07:30:59,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=439 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/dc1847fcae25409fad97132f001eaaf1 2024-11-11T07:30:59,483 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/145d67a6e51f463cba7ee3e63174c5d2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/145d67a6e51f463cba7ee3e63174c5d2 2024-11-11T07:30:59,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/145d67a6e51f463cba7ee3e63174c5d2, entries=250, sequenceid=439, filesize=47.5 K 2024-11-11T07:30:59,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:59,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310319485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:59,499 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310319485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:59,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:59,500 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:59,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310319485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:59,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:59,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310319487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:59,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/7e751ffdd58241908977f210cf4846b6 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/7e751ffdd58241908977f210cf4846b6 2024-11-11T07:30:59,505 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/7e751ffdd58241908977f210cf4846b6, entries=150, sequenceid=439, filesize=12.0 K 2024-11-11T07:30:59,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/dc1847fcae25409fad97132f001eaaf1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/dc1847fcae25409fad97132f001eaaf1 2024-11-11T07:30:59,516 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/dc1847fcae25409fad97132f001eaaf1, entries=150, sequenceid=439, filesize=12.0 K 2024-11-11T07:30:59,517 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6c7e70f5adf27c26564bc9bc160230a0 in 249ms, sequenceid=439, compaction requested=false 2024-11-11T07:30:59,517 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:30:59,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-11T07:30:59,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:30:59,596 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T07:30:59,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:30:59,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:59,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:30:59,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:59,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:30:59,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:30:59,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T07:30:59,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111221649433cc846a79a4419a3e8224871_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310259356/Put/seqid=0 2024-11-11T07:30:59,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742291_1467 (size=12454) 2024-11-11T07:30:59,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:30:59,666 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#A#compaction#389 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:30:59,666 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/2112ee267acc41429a2766fccd3bdfbd is 175, key is test_row_0/A:col10/1731310257885/Put/seqid=0 2024-11-11T07:30:59,670 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111221649433cc846a79a4419a3e8224871_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111221649433cc846a79a4419a3e8224871_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:59,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/c0f706b9cc0b49e2b556cd0353320230, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:30:59,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/c0f706b9cc0b49e2b556cd0353320230 is 175, key is test_row_0/A:col10/1731310259356/Put/seqid=0 2024-11-11T07:30:59,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742292_1468 (size=32175) 2024-11-11T07:30:59,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742293_1469 (size=31255) 2024-11-11T07:30:59,702 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=457, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/c0f706b9cc0b49e2b556cd0353320230 2024-11-11T07:30:59,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/55cfca0f0916428988f3f651735bf235 is 50, key is test_row_0/B:col10/1731310259356/Put/seqid=0 2024-11-11T07:30:59,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:30:59,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:30:59,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742294_1470 (size=12301) 2024-11-11T07:30:59,781 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:59,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310319760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:59,781 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:59,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:59,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310319768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:59,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,790 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:59,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310319770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:59,795 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310319771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:59,902 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:59,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310319885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:59,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,903 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:59,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:59,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310319895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,912 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:59,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310319905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:59,925 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:30:59,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310319905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:30:59,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,926 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:30:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:30:59,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,104 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/2112ee267acc41429a2766fccd3bdfbd as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/2112ee267acc41429a2766fccd3bdfbd 2024-11-11T07:31:00,109 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/A of 6c7e70f5adf27c26564bc9bc160230a0 into 2112ee267acc41429a2766fccd3bdfbd(size=31.4 K), total size for store is 78.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:00,109 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:00,109 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/A, priority=13, startTime=1731310259148; duration=0sec 2024-11-11T07:31:00,109 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:00,109 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:A 2024-11-11T07:31:00,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T07:31:00,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:00,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310320116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:00,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:00,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310320120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,125 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310320135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:00,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:00,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310320135, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:00,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,156 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,172 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/55cfca0f0916428988f3f651735bf235 2024-11-11T07:31:00,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/627edce4cf8f438d8aadeab3de9e9368 is 50, key is test_row_0/C:col10/1731310259356/Put/seqid=0 2024-11-11T07:31:00,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742295_1471 (size=12301) 2024-11-11T07:31:00,228 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=457 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/627edce4cf8f438d8aadeab3de9e9368 2024-11-11T07:31:00,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/c0f706b9cc0b49e2b556cd0353320230 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/c0f706b9cc0b49e2b556cd0353320230 2024-11-11T07:31:00,236 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/c0f706b9cc0b49e2b556cd0353320230, entries=150, sequenceid=457, filesize=30.5 K 2024-11-11T07:31:00,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/55cfca0f0916428988f3f651735bf235 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/55cfca0f0916428988f3f651735bf235 2024-11-11T07:31:00,240 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/55cfca0f0916428988f3f651735bf235, entries=150, sequenceid=457, filesize=12.0 K 2024-11-11T07:31:00,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/627edce4cf8f438d8aadeab3de9e9368 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/627edce4cf8f438d8aadeab3de9e9368 2024-11-11T07:31:00,246 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/627edce4cf8f438d8aadeab3de9e9368, entries=150, sequenceid=457, filesize=12.0 K 2024-11-11T07:31:00,247 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 6c7e70f5adf27c26564bc9bc160230a0 in 651ms, sequenceid=457, compaction requested=true 2024-11-11T07:31:00,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:00,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:00,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-11T07:31:00,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-11T07:31:00,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-11T07:31:00,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2710 sec 2024-11-11T07:31:00,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.2810 sec 2024-11-11T07:31:00,439 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T07:31:00,439 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:31:00,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:00,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:31:00,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:00,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:31:00,440 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:00,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:00,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110985170864724bf685cdd61a5d7cb7b8_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310260438/Put/seqid=0 2024-11-11T07:31:00,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742296_1472 (size=14994) 2024-11-11T07:31:00,453 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:00,456 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110985170864724bf685cdd61a5d7cb7b8_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110985170864724bf685cdd61a5d7cb7b8_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:00,457 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/7cf0baa47724416eaa9f729759ec3086, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:00,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/7cf0baa47724416eaa9f729759ec3086 is 175, key is test_row_0/A:col10/1731310260438/Put/seqid=0 2024-11-11T07:31:00,477 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742297_1473 (size=39949) 2024-11-11T07:31:00,477 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=474, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/7cf0baa47724416eaa9f729759ec3086 2024-11-11T07:31:00,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/b862ce7d46314147a2d2d62c4225eb71 is 50, key is test_row_0/B:col10/1731310260438/Put/seqid=0 2024-11-11T07:31:00,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742298_1474 (size=12301) 2024-11-11T07:31:00,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310320518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:00,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310320514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,544 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310320542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,563 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310320545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:00,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,566 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310320656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:00,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:00,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310320656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,667 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310320674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:00,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:00,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310320675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:00,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310320875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310320875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310320894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,901 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:00,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310320896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:00,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/b862ce7d46314147a2d2d62c4225eb71 2024-11-11T07:31:00,920 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/427c1a5cb44147f39126649d1d3382b2 is 50, key is test_row_0/C:col10/1731310260438/Put/seqid=0 2024-11-11T07:31:00,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742299_1475 (size=12301) 2024-11-11T07:31:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-11T07:31:01,137 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:01,138 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:01,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-11T07:31:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T07:31:01,156 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:01,158 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:01,158 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:01,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310321194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:01,205 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:01,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310321194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,206 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:01,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:01,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310321207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:01,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310321207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,220 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:01,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T07:31:01,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T07:31:01,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:01,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:31:01,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:01,312 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:01,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:01,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:01,369 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=474 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/427c1a5cb44147f39126649d1d3382b2 2024-11-11T07:31:01,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/7cf0baa47724416eaa9f729759ec3086 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/7cf0baa47724416eaa9f729759ec3086 2024-11-11T07:31:01,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/7cf0baa47724416eaa9f729759ec3086, entries=200, sequenceid=474, filesize=39.0 K 2024-11-11T07:31:01,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/b862ce7d46314147a2d2d62c4225eb71 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/b862ce7d46314147a2d2d62c4225eb71 2024-11-11T07:31:01,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/b862ce7d46314147a2d2d62c4225eb71, entries=150, sequenceid=474, filesize=12.0 K 2024-11-11T07:31:01,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/427c1a5cb44147f39126649d1d3382b2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/427c1a5cb44147f39126649d1d3382b2 2024-11-11T07:31:01,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/427c1a5cb44147f39126649d1d3382b2, entries=150, sequenceid=474, filesize=12.0 K 2024-11-11T07:31:01,390 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 6c7e70f5adf27c26564bc9bc160230a0 in 951ms, sequenceid=474, compaction requested=true 2024-11-11T07:31:01,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:01,390 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:01,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:01,391 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:01,391 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:01,392 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 152018 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:01,392 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/A is initiating minor compaction (all files) 2024-11-11T07:31:01,392 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/A in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:01,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:01,392 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/2112ee267acc41429a2766fccd3bdfbd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/145d67a6e51f463cba7ee3e63174c5d2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/c0f706b9cc0b49e2b556cd0353320230, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/7cf0baa47724416eaa9f729759ec3086] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=148.5 K 2024-11-11T07:31:01,392 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:01,392 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/2112ee267acc41429a2766fccd3bdfbd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/145d67a6e51f463cba7ee3e63174c5d2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/c0f706b9cc0b49e2b556cd0353320230, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/7cf0baa47724416eaa9f729759ec3086] 2024-11-11T07:31:01,392 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:01,392 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/B is initiating minor compaction (all files) 2024-11-11T07:31:01,392 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/B in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:01,392 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/11f4c2ecc27f4573871723fd978e0156, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/7e751ffdd58241908977f210cf4846b6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/55cfca0f0916428988f3f651735bf235, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/b862ce7d46314147a2d2d62c4225eb71] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=48.9 K 2024-11-11T07:31:01,393 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2112ee267acc41429a2766fccd3bdfbd, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731310257803 2024-11-11T07:31:01,393 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 11f4c2ecc27f4573871723fd978e0156, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731310257803 2024-11-11T07:31:01,393 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 145d67a6e51f463cba7ee3e63174c5d2, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731310258577 2024-11-11T07:31:01,393 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e751ffdd58241908977f210cf4846b6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731310258577 2024-11-11T07:31:01,394 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0f706b9cc0b49e2b556cd0353320230, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731310259356 2024-11-11T07:31:01,394 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cf0baa47724416eaa9f729759ec3086, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731310259756 2024-11-11T07:31:01,394 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 55cfca0f0916428988f3f651735bf235, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731310259356 2024-11-11T07:31:01,394 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting b862ce7d46314147a2d2d62c4225eb71, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731310259756 2024-11-11T07:31:01,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:01,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:01,395 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:01,411 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:01,416 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#B#compaction#401 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:01,416 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/040bb5f3b4c64c71b693289ff4c30743 is 50, key is test_row_0/B:col10/1731310260438/Put/seqid=0 2024-11-11T07:31:01,417 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111e8d427583aec48c584e8c189eeef5c61_6c7e70f5adf27c26564bc9bc160230a0 store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:01,421 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111e8d427583aec48c584e8c189eeef5c61_6c7e70f5adf27c26564bc9bc160230a0, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:01,421 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111e8d427583aec48c584e8c189eeef5c61_6c7e70f5adf27c26564bc9bc160230a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:01,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742300_1476 (size=13357) 2024-11-11T07:31:01,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742301_1477 (size=4469) 2024-11-11T07:31:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T07:31:01,466 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#A#compaction#400 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:01,466 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/911a017fa8b5403f806835a0e5328ad5 is 175, key is test_row_0/A:col10/1731310260438/Put/seqid=0 2024-11-11T07:31:01,470 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/040bb5f3b4c64c71b693289ff4c30743 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/040bb5f3b4c64c71b693289ff4c30743 2024-11-11T07:31:01,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-11T07:31:01,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:01,471 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-11T07:31:01,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:31:01,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:01,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:31:01,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:01,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:31:01,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:01,477 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/B of 6c7e70f5adf27c26564bc9bc160230a0 into 040bb5f3b4c64c71b693289ff4c30743(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:01,477 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:01,477 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/B, priority=12, startTime=1731310261391; duration=0sec 2024-11-11T07:31:01,477 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:01,477 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:B 2024-11-11T07:31:01,477 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:01,478 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:01,478 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/C is initiating minor compaction (all files) 2024-11-11T07:31:01,478 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/C in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:01,478 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/cbdd7d8cda5a41c3a39cbfe2f810cfa8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/dc1847fcae25409fad97132f001eaaf1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/627edce4cf8f438d8aadeab3de9e9368, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/427c1a5cb44147f39126649d1d3382b2] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=48.9 K 2024-11-11T07:31:01,479 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting cbdd7d8cda5a41c3a39cbfe2f810cfa8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731310257803 2024-11-11T07:31:01,479 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting dc1847fcae25409fad97132f001eaaf1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=439, earliestPutTs=1731310258577 2024-11-11T07:31:01,479 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 627edce4cf8f438d8aadeab3de9e9368, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=457, earliestPutTs=1731310259356 2024-11-11T07:31:01,479 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 427c1a5cb44147f39126649d1d3382b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731310259756 2024-11-11T07:31:01,495 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742302_1478 (size=32311) 2024-11-11T07:31:01,501 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/911a017fa8b5403f806835a0e5328ad5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/911a017fa8b5403f806835a0e5328ad5 2024-11-11T07:31:01,506 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/A of 6c7e70f5adf27c26564bc9bc160230a0 into 911a017fa8b5403f806835a0e5328ad5(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:01,506 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:01,507 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/A, priority=12, startTime=1731310261390; duration=0sec 2024-11-11T07:31:01,507 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:01,507 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:A 2024-11-11T07:31:01,508 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#C#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:01,508 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/f673aa2a6f4b4e41b337b57dab1033eb is 50, key is test_row_0/C:col10/1731310260438/Put/seqid=0 2024-11-11T07:31:01,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111164c6c222d684846aaacd74b3805bbd7_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310260515/Put/seqid=0 2024-11-11T07:31:01,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742303_1479 (size=13323) 2024-11-11T07:31:01,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742304_1480 (size=12454) 2024-11-11T07:31:01,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:01,555 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111164c6c222d684846aaacd74b3805bbd7_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111164c6c222d684846aaacd74b3805bbd7_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:01,557 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/f673aa2a6f4b4e41b337b57dab1033eb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f673aa2a6f4b4e41b337b57dab1033eb 2024-11-11T07:31:01,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/199fbf25b2904851bccdee16a6015db5, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:01,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/199fbf25b2904851bccdee16a6015db5 is 175, key is test_row_0/A:col10/1731310260515/Put/seqid=0 2024-11-11T07:31:01,563 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/C of 6c7e70f5adf27c26564bc9bc160230a0 into f673aa2a6f4b4e41b337b57dab1033eb(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:01,563 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:01,563 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/C, priority=12, startTime=1731310261395; duration=0sec 2024-11-11T07:31:01,563 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:01,563 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:C 2024-11-11T07:31:01,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742305_1481 (size=31255) 2024-11-11T07:31:01,576 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=490, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/199fbf25b2904851bccdee16a6015db5 2024-11-11T07:31:01,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/a1fe5098b8e540e18b7b1ac4806cbe28 is 50, key is test_row_0/B:col10/1731310260515/Put/seqid=0 2024-11-11T07:31:01,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742306_1482 (size=12301) 2024-11-11T07:31:01,647 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/a1fe5098b8e540e18b7b1ac4806cbe28 2024-11-11T07:31:01,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/ee4ac3af77b54fc0849ae8b543308702 is 50, key is test_row_0/C:col10/1731310260515/Put/seqid=0 2024-11-11T07:31:01,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:01,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:31:01,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742307_1483 (size=12301) 2024-11-11T07:31:01,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T07:31:01,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:01,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310321768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,782 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:01,782 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:01,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310321769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:01,795 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:01,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310321780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:01,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:01,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:01,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310321781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:01,890 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:01,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310321885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:01,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,891 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:01,909 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:01,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310321895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:01,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,910 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:01,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:01,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310321909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:01,917 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310321909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:01,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,103 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310322095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:02,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,103 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,104 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310322115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310322124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:02,133 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310322124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,135 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,138 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/ee4ac3af77b54fc0849ae8b543308702 2024-11-11T07:31:02,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/199fbf25b2904851bccdee16a6015db5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/199fbf25b2904851bccdee16a6015db5 2024-11-11T07:31:02,150 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/199fbf25b2904851bccdee16a6015db5, entries=150, sequenceid=490, filesize=30.5 K 2024-11-11T07:31:02,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/a1fe5098b8e540e18b7b1ac4806cbe28 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/a1fe5098b8e540e18b7b1ac4806cbe28 2024-11-11T07:31:02,154 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/a1fe5098b8e540e18b7b1ac4806cbe28, entries=150, sequenceid=490, filesize=12.0 K 2024-11-11T07:31:02,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/ee4ac3af77b54fc0849ae8b543308702 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ee4ac3af77b54fc0849ae8b543308702 2024-11-11T07:31:02,158 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ee4ac3af77b54fc0849ae8b543308702, entries=150, sequenceid=490, filesize=12.0 K 2024-11-11T07:31:02,159 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 6c7e70f5adf27c26564bc9bc160230a0 in 688ms, sequenceid=490, compaction requested=false 2024-11-11T07:31:02,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:02,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:02,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-11T07:31:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-11T07:31:02,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-11T07:31:02,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0030 sec 2024-11-11T07:31:02,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.0250 sec 2024-11-11T07:31:02,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-11T07:31:02,289 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:02,290 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-11T07:31:02,293 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T07:31:02,294 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:02,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T07:31:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:02,423 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T07:31:02,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:31:02,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:02,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:31:02,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:02,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:31:02,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:02,437 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111146a97ce439224912966bdf48619bb18c_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310261779/Put/seqid=0 2024-11-11T07:31:02,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T07:31:02,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:02,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:31:02,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:02,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:02,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:02,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:02,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742308_1484 (size=14994) 2024-11-11T07:31:02,461 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:02,464 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111146a97ce439224912966bdf48619bb18c_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111146a97ce439224912966bdf48619bb18c_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:02,465 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/b896f222ab164a2cafeeb5fd72bed93a, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:02,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/b896f222ab164a2cafeeb5fd72bed93a is 175, key is test_row_0/A:col10/1731310261779/Put/seqid=0 2024-11-11T07:31:02,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742309_1485 (size=39949) 2024-11-11T07:31:02,474 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=510, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/b896f222ab164a2cafeeb5fd72bed93a 2024-11-11T07:31:02,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/9db0703b074d400c83e4173cb1180eeb is 50, key is test_row_0/B:col10/1731310261779/Put/seqid=0 2024-11-11T07:31:02,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742310_1486 (size=12301) 2024-11-11T07:31:02,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310322533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:02,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,553 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:02,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310322534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:02,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,554 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:02,554 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310322539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:02,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,554 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:02,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310322552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T07:31:02,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:02,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:31:02,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:02,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:02,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:02,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T07:31:02,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:02,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310322666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:02,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310322666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:02,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:02,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310322667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,675 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:02,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310322675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,676 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T07:31:02,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:02,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:31:02,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:02,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:02,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:02,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:02,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/9db0703b074d400c83e4173cb1180eeb 2024-11-11T07:31:02,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310322885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:02,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:02,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310322885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:02,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310322885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:02,894 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:02,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310322885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:02,897 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:02,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T07:31:02,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T07:31:02,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:02,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:31:02,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:02,931 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:02,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:02,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:02,940 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/dc1d4ae7a03340c7ae966b2ad8df3fd3 is 50, key is test_row_0/C:col10/1731310261779/Put/seqid=0 2024-11-11T07:31:02,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742311_1487 (size=12301) 2024-11-11T07:31:02,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/dc1d4ae7a03340c7ae966b2ad8df3fd3 2024-11-11T07:31:02,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/b896f222ab164a2cafeeb5fd72bed93a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/b896f222ab164a2cafeeb5fd72bed93a 2024-11-11T07:31:02,957 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/b896f222ab164a2cafeeb5fd72bed93a, entries=200, sequenceid=510, filesize=39.0 K 2024-11-11T07:31:02,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/9db0703b074d400c83e4173cb1180eeb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/9db0703b074d400c83e4173cb1180eeb 2024-11-11T07:31:02,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/9db0703b074d400c83e4173cb1180eeb, entries=150, sequenceid=510, filesize=12.0 K 2024-11-11T07:31:02,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/dc1d4ae7a03340c7ae966b2ad8df3fd3 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/dc1d4ae7a03340c7ae966b2ad8df3fd3 2024-11-11T07:31:02,967 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/dc1d4ae7a03340c7ae966b2ad8df3fd3, entries=150, sequenceid=510, filesize=12.0 K 2024-11-11T07:31:02,968 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 6c7e70f5adf27c26564bc9bc160230a0 in 545ms, sequenceid=510, compaction requested=true 2024-11-11T07:31:02,968 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:02,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:02,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:02,969 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:02,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:02,969 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:02,972 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:02,972 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:02,973 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/A is initiating minor compaction (all files) 2024-11-11T07:31:02,973 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:02,973 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/A in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:02,973 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/B is initiating minor compaction (all files) 2024-11-11T07:31:02,973 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/B in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:02,973 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/911a017fa8b5403f806835a0e5328ad5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/199fbf25b2904851bccdee16a6015db5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/b896f222ab164a2cafeeb5fd72bed93a] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=101.1 K 2024-11-11T07:31:02,973 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/040bb5f3b4c64c71b693289ff4c30743, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/a1fe5098b8e540e18b7b1ac4806cbe28, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/9db0703b074d400c83e4173cb1180eeb] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=37.1 K 2024-11-11T07:31:02,973 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:02,973 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/911a017fa8b5403f806835a0e5328ad5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/199fbf25b2904851bccdee16a6015db5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/b896f222ab164a2cafeeb5fd72bed93a] 2024-11-11T07:31:02,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:02,973 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:02,973 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 040bb5f3b4c64c71b693289ff4c30743, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731310259756 2024-11-11T07:31:02,973 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 911a017fa8b5403f806835a0e5328ad5, keycount=150, bloomtype=ROW, size=31.6 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731310259756 2024-11-11T07:31:02,973 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting a1fe5098b8e540e18b7b1ac4806cbe28, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731310260515 2024-11-11T07:31:02,974 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 199fbf25b2904851bccdee16a6015db5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731310260515 2024-11-11T07:31:02,974 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 9db0703b074d400c83e4173cb1180eeb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731310261765 2024-11-11T07:31:02,974 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting b896f222ab164a2cafeeb5fd72bed93a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731310261764 2024-11-11T07:31:02,989 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#B#compaction#409 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:02,989 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/a1ae8d4bdb474e98935a37f384cd6b4c is 50, key is test_row_0/B:col10/1731310261779/Put/seqid=0 2024-11-11T07:31:02,999 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:03,011 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411116200d8cc421543be941dba9c87352eb8_6c7e70f5adf27c26564bc9bc160230a0 store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:03,013 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411116200d8cc421543be941dba9c87352eb8_6c7e70f5adf27c26564bc9bc160230a0, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:03,013 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411116200d8cc421543be941dba9c87352eb8_6c7e70f5adf27c26564bc9bc160230a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:03,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742313_1489 (size=4469) 2024-11-11T07:31:03,040 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#A#compaction#410 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:03,040 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/e03565e419c44e1eb3c08f915a0be7af is 175, key is test_row_0/A:col10/1731310261779/Put/seqid=0 2024-11-11T07:31:03,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742312_1488 (size=13459) 2024-11-11T07:31:03,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742314_1490 (size=32413) 2024-11-11T07:31:03,054 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/a1ae8d4bdb474e98935a37f384cd6b4c as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/a1ae8d4bdb474e98935a37f384cd6b4c 2024-11-11T07:31:03,059 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/e03565e419c44e1eb3c08f915a0be7af as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e03565e419c44e1eb3c08f915a0be7af 2024-11-11T07:31:03,065 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/B of 6c7e70f5adf27c26564bc9bc160230a0 into a1ae8d4bdb474e98935a37f384cd6b4c(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:03,066 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:03,066 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/B, priority=13, startTime=1731310262969; duration=0sec 2024-11-11T07:31:03,066 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:03,066 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:B 2024-11-11T07:31:03,066 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:03,069 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37925 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:03,069 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/C is initiating minor compaction (all files) 2024-11-11T07:31:03,069 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/C in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:03,069 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f673aa2a6f4b4e41b337b57dab1033eb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ee4ac3af77b54fc0849ae8b543308702, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/dc1d4ae7a03340c7ae966b2ad8df3fd3] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=37.0 K 2024-11-11T07:31:03,069 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/A of 6c7e70f5adf27c26564bc9bc160230a0 into e03565e419c44e1eb3c08f915a0be7af(size=31.7 K), total size for store is 31.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:03,069 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:03,069 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/A, priority=13, startTime=1731310262969; duration=0sec 2024-11-11T07:31:03,069 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:03,069 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:A 2024-11-11T07:31:03,069 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f673aa2a6f4b4e41b337b57dab1033eb, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=474, earliestPutTs=1731310259756 2024-11-11T07:31:03,070 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting ee4ac3af77b54fc0849ae8b543308702, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=490, earliestPutTs=1731310260515 2024-11-11T07:31:03,070 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting dc1d4ae7a03340c7ae966b2ad8df3fd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731310261765 2024-11-11T07:31:03,079 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#C#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:03,079 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/8b264563401a497387e3a37a576984c7 is 50, key is test_row_0/C:col10/1731310261779/Put/seqid=0 2024-11-11T07:31:03,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-11T07:31:03,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:03,087 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T07:31:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:31:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:31:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:31:03,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:03,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742315_1491 (size=13425) 2024-11-11T07:31:03,121 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/8b264563401a497387e3a37a576984c7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/8b264563401a497387e3a37a576984c7 2024-11-11T07:31:03,128 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/C of 6c7e70f5adf27c26564bc9bc160230a0 into 8b264563401a497387e3a37a576984c7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:03,128 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:03,128 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/C, priority=13, startTime=1731310262969; duration=0sec 2024-11-11T07:31:03,128 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:03,128 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:C 2024-11-11T07:31:03,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114c46567e50364617b3fc7fa9d9b29ccb_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310262527/Put/seqid=0 2024-11-11T07:31:03,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742316_1492 (size=12454) 2024-11-11T07:31:03,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:03,158 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114c46567e50364617b3fc7fa9d9b29ccb_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114c46567e50364617b3fc7fa9d9b29ccb_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:03,158 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/42b83905fd894944bb30a421b5736df3, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:03,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/42b83905fd894944bb30a421b5736df3 is 175, key is test_row_0/A:col10/1731310262527/Put/seqid=0 2024-11-11T07:31:03,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742317_1493 (size=31255) 2024-11-11T07:31:03,191 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=527, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/42b83905fd894944bb30a421b5736df3 2024-11-11T07:31:03,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/88bfe35323454217989f55ceaa2b1d34 is 50, key is test_row_0/B:col10/1731310262527/Put/seqid=0 2024-11-11T07:31:03,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742318_1494 (size=12301) 2024-11-11T07:31:03,210 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/88bfe35323454217989f55ceaa2b1d34 2024-11-11T07:31:03,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:03,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:31:03,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/00a4490cfe0b4260a0d00997e0e569ac is 50, key is test_row_0/C:col10/1731310262527/Put/seqid=0 2024-11-11T07:31:03,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742319_1495 (size=12301) 2024-11-11T07:31:03,255 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=527 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/00a4490cfe0b4260a0d00997e0e569ac 2024-11-11T07:31:03,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/42b83905fd894944bb30a421b5736df3 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/42b83905fd894944bb30a421b5736df3 2024-11-11T07:31:03,262 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/42b83905fd894944bb30a421b5736df3, entries=150, sequenceid=527, filesize=30.5 K 2024-11-11T07:31:03,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/88bfe35323454217989f55ceaa2b1d34 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/88bfe35323454217989f55ceaa2b1d34 2024-11-11T07:31:03,266 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/88bfe35323454217989f55ceaa2b1d34, entries=150, sequenceid=527, filesize=12.0 K 2024-11-11T07:31:03,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/00a4490cfe0b4260a0d00997e0e569ac as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/00a4490cfe0b4260a0d00997e0e569ac 2024-11-11T07:31:03,272 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/00a4490cfe0b4260a0d00997e0e569ac, entries=150, sequenceid=527, filesize=12.0 K 2024-11-11T07:31:03,272 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 6c7e70f5adf27c26564bc9bc160230a0 in 186ms, sequenceid=527, compaction requested=false 2024-11-11T07:31:03,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:03,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:03,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-11T07:31:03,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-11T07:31:03,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T07:31:03,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-11T07:31:03,276 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 980 msec 2024-11-11T07:31:03,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:31:03,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:03,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:31:03,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:03,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:31:03,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:03,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:03,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 986 msec 2024-11-11T07:31:03,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111df6e89b782c14865a4cc81e14245f366_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310263261/Put/seqid=0 2024-11-11T07:31:03,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742320_1496 (size=14994) 2024-11-11T07:31:03,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:03,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310323310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:03,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310323313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:03,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:03,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310323313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,325 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:03,326 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:03,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310323316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,327 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:03,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-11T07:31:03,435 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:03,436 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:03,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-11T07:31:03,437 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T07:31:03,438 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:03,438 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:03,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310323435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:03,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310323436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:03,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:03,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310323436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:03,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,450 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:03,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:03,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310323435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,451 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:03,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T07:31:03,591 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T07:31:03,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:03,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:31:03,591 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:03,591 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:03,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:03,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:03,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310323654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:03,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310323655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:03,668 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310323655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310323655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:03,708 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:03,711 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111df6e89b782c14865a4cc81e14245f366_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111df6e89b782c14865a4cc81e14245f366_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:03,712 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/0578e0a1b97d40bc97ea5a06dabcb251, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:03,712 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/0578e0a1b97d40bc97ea5a06dabcb251 is 175, key is test_row_0/A:col10/1731310263261/Put/seqid=0 2024-11-11T07:31:03,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T07:31:03,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:03,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:31:03,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:03,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:03,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:03,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:03,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742321_1497 (size=39949) 2024-11-11T07:31:03,752 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=546, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/0578e0a1b97d40bc97ea5a06dabcb251 2024-11-11T07:31:03,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T07:31:03,770 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/dbf8dc79c86e4ee7b9e20ed57d690ea1 is 50, key is test_row_0/B:col10/1731310263261/Put/seqid=0 2024-11-11T07:31:03,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742322_1498 (size=12301) 2024-11-11T07:31:03,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/dbf8dc79c86e4ee7b9e20ed57d690ea1 2024-11-11T07:31:03,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T07:31:03,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:03,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:31:03,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:03,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:03,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:03,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:03,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/015289fdb97c4a84900f2031f2b2298f is 50, key is test_row_0/C:col10/1731310263261/Put/seqid=0 2024-11-11T07:31:03,908 INFO [Thread-1611 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1611. 2024-11-11T07:31:03,908 DEBUG [Thread-1611 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:03,908 DEBUG [Thread-1611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:03,908 DEBUG [Thread-1611 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:03,908 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:03,911 INFO [Thread-1609 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1609. 2024-11-11T07:31:03,911 DEBUG [Thread-1609 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:03,911 DEBUG [Thread-1609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:03,911 DEBUG [Thread-1609 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:03,911 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:03,912 INFO [Thread-1613 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1613. 2024-11-11T07:31:03,912 DEBUG [Thread-1613 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:03,912 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:03,913 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:03,913 DEBUG [Thread-1613 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:03,924 INFO [Thread-1605 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1605. 2024-11-11T07:31:03,924 DEBUG [Thread-1605 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:03,924 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:03,924 DEBUG [Thread-1605 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:03,924 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:03,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742323_1499 (size=12301) 2024-11-11T07:31:03,926 INFO [Thread-1607 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1607. 2024-11-11T07:31:03,926 DEBUG [Thread-1607 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:03,926 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:03,926 DEBUG [Thread-1607 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:03,926 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:03,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56474 deadline: 1731310323975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:03,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56554 deadline: 1731310323975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:03,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:03,976 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56546 deadline: 1731310323975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:03,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:03,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56522 deadline: 1731310323975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:03,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:04,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T07:31:04,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:04,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:31:04,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:04,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:04,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:04,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:04,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T07:31:04,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:04,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:56486 deadline: 1731310324165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:04,166 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_0' in 6c7e70f5adf27c26564bc9bc160230a0 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18431 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:31:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:04,166 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:04,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T07:31:04,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:04,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:31:04,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:04,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:04,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:04,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:04,325 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=546 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/015289fdb97c4a84900f2031f2b2298f 2024-11-11T07:31:04,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/0578e0a1b97d40bc97ea5a06dabcb251 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/0578e0a1b97d40bc97ea5a06dabcb251 2024-11-11T07:31:04,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/0578e0a1b97d40bc97ea5a06dabcb251, entries=200, sequenceid=546, filesize=39.0 K 2024-11-11T07:31:04,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/dbf8dc79c86e4ee7b9e20ed57d690ea1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/dbf8dc79c86e4ee7b9e20ed57d690ea1 2024-11-11T07:31:04,335 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/dbf8dc79c86e4ee7b9e20ed57d690ea1, entries=150, sequenceid=546, filesize=12.0 K 2024-11-11T07:31:04,336 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/015289fdb97c4a84900f2031f2b2298f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/015289fdb97c4a84900f2031f2b2298f 2024-11-11T07:31:04,339 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/015289fdb97c4a84900f2031f2b2298f, entries=150, sequenceid=546, filesize=12.0 K 2024-11-11T07:31:04,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6c7e70f5adf27c26564bc9bc160230a0 in 1064ms, sequenceid=546, compaction requested=true 2024-11-11T07:31:04,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:04,340 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:04,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:04,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:04,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:04,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:04,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6c7e70f5adf27c26564bc9bc160230a0:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:04,340 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:04,340 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:04,340 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103617 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:04,340 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/A is initiating minor compaction (all files) 2024-11-11T07:31:04,340 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38061 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:04,340 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/A in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:04,340 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/B is initiating minor compaction (all files) 2024-11-11T07:31:04,340 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/B in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:04,341 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e03565e419c44e1eb3c08f915a0be7af, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/42b83905fd894944bb30a421b5736df3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/0578e0a1b97d40bc97ea5a06dabcb251] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=101.2 K 2024-11-11T07:31:04,341 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:04,341 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/a1ae8d4bdb474e98935a37f384cd6b4c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/88bfe35323454217989f55ceaa2b1d34, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/dbf8dc79c86e4ee7b9e20ed57d690ea1] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=37.2 K 2024-11-11T07:31:04,341 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e03565e419c44e1eb3c08f915a0be7af, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/42b83905fd894944bb30a421b5736df3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/0578e0a1b97d40bc97ea5a06dabcb251] 2024-11-11T07:31:04,341 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting a1ae8d4bdb474e98935a37f384cd6b4c, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731310261765 2024-11-11T07:31:04,341 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting e03565e419c44e1eb3c08f915a0be7af, keycount=150, bloomtype=ROW, size=31.7 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731310261765 2024-11-11T07:31:04,341 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 88bfe35323454217989f55ceaa2b1d34, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731310262527 2024-11-11T07:31:04,341 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42b83905fd894944bb30a421b5736df3, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731310262527 2024-11-11T07:31:04,341 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting dbf8dc79c86e4ee7b9e20ed57d690ea1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1731310263246 2024-11-11T07:31:04,341 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0578e0a1b97d40bc97ea5a06dabcb251, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1731310263246 2024-11-11T07:31:04,348 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#B#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:04,349 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/b9783ca4228c42a2844ed866c6faf3e6 is 50, key is test_row_0/B:col10/1731310263261/Put/seqid=0 2024-11-11T07:31:04,352 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:04,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742324_1500 (size=13561) 2024-11-11T07:31:04,354 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111ce136c0f0c3a477f82d8ecd7baf5e271_6c7e70f5adf27c26564bc9bc160230a0 store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:04,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-11T07:31:04,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:04,357 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T07:31:04,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:31:04,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:04,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:31:04,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:04,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:31:04,357 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:04,359 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/b9783ca4228c42a2844ed866c6faf3e6 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/b9783ca4228c42a2844ed866c6faf3e6 2024-11-11T07:31:04,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111830a1aec1670418a80e3191e7ad0c972_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310263297/Put/seqid=0 2024-11-11T07:31:04,367 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/B of 6c7e70f5adf27c26564bc9bc160230a0 into b9783ca4228c42a2844ed866c6faf3e6(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:04,367 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:04,367 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/B, priority=13, startTime=1731310264340; duration=0sec 2024-11-11T07:31:04,367 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:04,367 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:B 2024-11-11T07:31:04,367 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:04,368 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38027 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:04,368 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 6c7e70f5adf27c26564bc9bc160230a0/C is initiating minor compaction (all files) 2024-11-11T07:31:04,368 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6c7e70f5adf27c26564bc9bc160230a0/C in TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:04,368 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/8b264563401a497387e3a37a576984c7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/00a4490cfe0b4260a0d00997e0e569ac, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/015289fdb97c4a84900f2031f2b2298f] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp, totalSize=37.1 K 2024-11-11T07:31:04,369 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 8b264563401a497387e3a37a576984c7, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731310261765 2024-11-11T07:31:04,369 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 00a4490cfe0b4260a0d00997e0e569ac, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=527, earliestPutTs=1731310262527 2024-11-11T07:31:04,369 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 015289fdb97c4a84900f2031f2b2298f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=546, earliestPutTs=1731310263246 2024-11-11T07:31:04,373 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111ce136c0f0c3a477f82d8ecd7baf5e271_6c7e70f5adf27c26564bc9bc160230a0, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:04,373 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111ce136c0f0c3a477f82d8ecd7baf5e271_6c7e70f5adf27c26564bc9bc160230a0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:04,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742325_1501 (size=12454) 2024-11-11T07:31:04,385 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#C#compaction#421 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:04,386 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/16dbb1501c59485daa3a50d6caf613f2 is 50, key is test_row_0/C:col10/1731310263261/Put/seqid=0 2024-11-11T07:31:04,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742326_1502 (size=4469) 2024-11-11T07:31:04,392 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6c7e70f5adf27c26564bc9bc160230a0#A#compaction#419 average throughput is 0.61 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:04,393 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/1db0a7bce92c4eee9fcfdb4917c0d49a is 175, key is test_row_0/A:col10/1731310263261/Put/seqid=0 2024-11-11T07:31:04,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742328_1504 (size=32515) 2024-11-11T07:31:04,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742327_1503 (size=13527) 2024-11-11T07:31:04,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:04,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. as already flushing 2024-11-11T07:31:04,488 INFO [Thread-1594 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1594. 2024-11-11T07:31:04,488 DEBUG [Thread-1594 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:04,488 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:04,488 DEBUG [Thread-1594 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:04,488 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:04,493 INFO [Thread-1602 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1602. 2024-11-11T07:31:04,493 DEBUG [Thread-1602 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:04,493 DEBUG [Thread-1602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:04,493 DEBUG [Thread-1602 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:04,493 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:04,494 INFO [Thread-1596 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1596. 2024-11-11T07:31:04,494 INFO [Thread-1600 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1600. 2024-11-11T07:31:04,494 DEBUG [Thread-1596 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:04,494 DEBUG [Thread-1600 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:04,494 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:04,494 DEBUG [Thread-1600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:04,495 DEBUG [Thread-1596 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:04,495 DEBUG [Thread-1600 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:04,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:04,495 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:04,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T07:31:04,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:04,779 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111830a1aec1670418a80e3191e7ad0c972_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111830a1aec1670418a80e3191e7ad0c972_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:04,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/f0dddb0c8d5649b2a1441920979c62aa, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:04,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/f0dddb0c8d5649b2a1441920979c62aa is 175, key is test_row_0/A:col10/1731310263297/Put/seqid=0 2024-11-11T07:31:04,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742329_1505 (size=31255) 2024-11-11T07:31:04,813 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/1db0a7bce92c4eee9fcfdb4917c0d49a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1db0a7bce92c4eee9fcfdb4917c0d49a 2024-11-11T07:31:04,813 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/16dbb1501c59485daa3a50d6caf613f2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/16dbb1501c59485daa3a50d6caf613f2 2024-11-11T07:31:04,818 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/C of 6c7e70f5adf27c26564bc9bc160230a0 into 16dbb1501c59485daa3a50d6caf613f2(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:04,818 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:04,818 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/C, priority=13, startTime=1731310264340; duration=0sec 2024-11-11T07:31:04,818 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6c7e70f5adf27c26564bc9bc160230a0/A of 6c7e70f5adf27c26564bc9bc160230a0 into 1db0a7bce92c4eee9fcfdb4917c0d49a(size=31.8 K), total size for store is 31.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:04,818 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:04,818 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., storeName=6c7e70f5adf27c26564bc9bc160230a0/A, priority=13, startTime=1731310264339; duration=0sec 2024-11-11T07:31:04,818 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:04,818 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:C 2024-11-11T07:31:04,818 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:04,818 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6c7e70f5adf27c26564bc9bc160230a0:A 2024-11-11T07:31:05,184 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=560, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/f0dddb0c8d5649b2a1441920979c62aa 2024-11-11T07:31:05,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/f07c8dd2c4da4eeaa5e6213aa9d02f70 is 50, key is test_row_0/B:col10/1731310263297/Put/seqid=0 2024-11-11T07:31:05,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742330_1506 (size=12301) 2024-11-11T07:31:05,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T07:31:05,594 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/f07c8dd2c4da4eeaa5e6213aa9d02f70 2024-11-11T07:31:05,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/555c3811855343fd9f774a87addd4fb7 is 50, key is test_row_0/C:col10/1731310263297/Put/seqid=0 2024-11-11T07:31:05,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742331_1507 (size=12301) 2024-11-11T07:31:06,004 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=560 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/555c3811855343fd9f774a87addd4fb7 2024-11-11T07:31:06,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/f0dddb0c8d5649b2a1441920979c62aa as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/f0dddb0c8d5649b2a1441920979c62aa 2024-11-11T07:31:06,010 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/f0dddb0c8d5649b2a1441920979c62aa, entries=150, sequenceid=560, filesize=30.5 K 2024-11-11T07:31:06,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/f07c8dd2c4da4eeaa5e6213aa9d02f70 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/f07c8dd2c4da4eeaa5e6213aa9d02f70 2024-11-11T07:31:06,013 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/f07c8dd2c4da4eeaa5e6213aa9d02f70, entries=150, sequenceid=560, filesize=12.0 K 2024-11-11T07:31:06,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/555c3811855343fd9f774a87addd4fb7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/555c3811855343fd9f774a87addd4fb7 2024-11-11T07:31:06,016 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/555c3811855343fd9f774a87addd4fb7, entries=150, sequenceid=560, filesize=12.0 K 2024-11-11T07:31:06,017 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=26.84 KB/27480 for 6c7e70f5adf27c26564bc9bc160230a0 in 1659ms, sequenceid=560, compaction requested=false 2024-11-11T07:31:06,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 6c7e70f5adf27c26564bc9bc160230a0: 2024-11-11T07:31:06,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:06,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-11T07:31:06,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-11T07:31:06,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-11T07:31:06,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5800 sec 2024-11-11T07:31:06,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 2.5830 sec 2024-11-11T07:31:07,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-11T07:31:07,604 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:13,334 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T07:31:14,196 INFO [Thread-1598 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1598. 2024-11-11T07:31:14,196 DEBUG [Thread-1598 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:14,196 DEBUG [Thread-1598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:14,196 DEBUG [Thread-1598 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:14,196 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 86 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 105 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 696 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2088 rows 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 698 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2094 rows 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 687 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2061 rows 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 690 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2070 rows 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 689 2024-11-11T07:31:14,198 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2067 rows 2024-11-11T07:31:14,198 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T07:31:14,198 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:31:14,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:14,198 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:14,198 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:14,199 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T07:31:14,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:14,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T07:31:14,202 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310274202"}]},"ts":"1731310274202"} 2024-11-11T07:31:14,203 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T07:31:14,203 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T07:31:14,204 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:31:14,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c7e70f5adf27c26564bc9bc160230a0, UNASSIGN}] 2024-11-11T07:31:14,206 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c7e70f5adf27c26564bc9bc160230a0, UNASSIGN 2024-11-11T07:31:14,206 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=6c7e70f5adf27c26564bc9bc160230a0, regionState=CLOSING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:14,208 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c7e70f5adf27c26564bc9bc160230a0, UNASSIGN because future has completed 2024-11-11T07:31:14,208 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:31:14,208 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:31:14,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T07:31:14,360 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(122): Close 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:14,360 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:31:14,360 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1722): Closing 6c7e70f5adf27c26564bc9bc160230a0, disabling compactions & flushes 2024-11-11T07:31:14,360 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:14,360 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:14,360 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. after waiting 0 ms 2024-11-11T07:31:14,360 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:14,360 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(2902): Flushing 6c7e70f5adf27c26564bc9bc160230a0 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-11T07:31:14,361 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=A 2024-11-11T07:31:14,361 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:14,361 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=B 2024-11-11T07:31:14,361 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:14,361 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6c7e70f5adf27c26564bc9bc160230a0, store=C 2024-11-11T07:31:14,361 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:14,366 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111170863f9697ca46c5822de1ee1b9397cf_6c7e70f5adf27c26564bc9bc160230a0 is 50, key is test_row_0/A:col10/1731310274195/Put/seqid=0 2024-11-11T07:31:14,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742332_1508 (size=9914) 2024-11-11T07:31:14,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T07:31:14,769 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:14,772 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111170863f9697ca46c5822de1ee1b9397cf_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111170863f9697ca46c5822de1ee1b9397cf_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:14,773 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/c2a05e412581497da7809c2305578c9f, store: [table=TestAcidGuarantees family=A region=6c7e70f5adf27c26564bc9bc160230a0] 2024-11-11T07:31:14,774 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/c2a05e412581497da7809c2305578c9f is 175, key is test_row_0/A:col10/1731310274195/Put/seqid=0 2024-11-11T07:31:14,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742333_1509 (size=22561) 2024-11-11T07:31:14,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T07:31:15,183 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=571, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/c2a05e412581497da7809c2305578c9f 2024-11-11T07:31:15,188 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/ef007f379d3f4739abd640134f34e1b9 is 50, key is test_row_0/B:col10/1731310274195/Put/seqid=0 2024-11-11T07:31:15,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742334_1510 (size=9857) 2024-11-11T07:31:15,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T07:31:15,595 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/ef007f379d3f4739abd640134f34e1b9 2024-11-11T07:31:15,601 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/d4486bb0a68e4e28b9614225c880f869 is 50, key is test_row_0/C:col10/1731310274195/Put/seqid=0 2024-11-11T07:31:15,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742335_1511 (size=9857) 2024-11-11T07:31:16,008 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/d4486bb0a68e4e28b9614225c880f869 2024-11-11T07:31:16,012 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/A/c2a05e412581497da7809c2305578c9f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/c2a05e412581497da7809c2305578c9f 2024-11-11T07:31:16,015 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/c2a05e412581497da7809c2305578c9f, entries=100, sequenceid=571, filesize=22.0 K 2024-11-11T07:31:16,015 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/B/ef007f379d3f4739abd640134f34e1b9 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/ef007f379d3f4739abd640134f34e1b9 2024-11-11T07:31:16,019 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/ef007f379d3f4739abd640134f34e1b9, entries=100, sequenceid=571, filesize=9.6 K 2024-11-11T07:31:16,019 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/.tmp/C/d4486bb0a68e4e28b9614225c880f869 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d4486bb0a68e4e28b9614225c880f869 2024-11-11T07:31:16,023 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d4486bb0a68e4e28b9614225c880f869, entries=100, sequenceid=571, filesize=9.6 K 2024-11-11T07:31:16,023 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6c7e70f5adf27c26564bc9bc160230a0 in 1663ms, sequenceid=571, compaction requested=true 2024-11-11T07:31:16,024 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/f185e5e9cf7a4dc492ccdd9c4c0788d6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/787df8e8f6c4472aade64eb76ea5d41d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/251dd0d4fc7943d7bfaa2c9ff67598bf, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/9becb4d43162463ab42b704162a4d30b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/76a0b9a88a6a4e9c921d026abdaa34e2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/01699738f6c746919c58a86763a36712, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/12e3aa9c7a68499a9dcb8115268a35bb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/58cb4f093b7e4dfb82d07ee6b984ce2b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/54706287047448919decb07c60f827f2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e9fd86eefde34c97804dbb3a918ae00a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/a08a80af5bf2433da183d69fbc2285c8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1336199e218b4c929511eeb73ad20a2d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/b2cf42e07fc149a1b736714c9e2719c8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/8d9991d938984d43bff72019c0f4ef80, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/3a5d5022ab414540b816d32a126c79a5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/13589f1d1e3f4aa59877cdd03c83d3cc, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1902c68ade564831ae3812d4b2de9395, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/301e552ddd7f44ed9da3a9d6919fdfdb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/0d9a92eccfcd4cc9b86d958a8b3451ce, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/5b3ccae13ee845fea3a7c295b40fc8d1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/940e59b19620410cbbaaae0f1da8dd9f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/a41e6ccf90074d768c592d8bc52fe698, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e417d9b9d73f4a059ca946b0e7baabe1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/d6f40a8b633b46d6bfe3e5c8496813c1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/617eb9bcdc684ae290970659785dff91, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e3541e505178446b8132476d874f3e0e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/148f0628487743bca49fbe846c4b27d3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/2112ee267acc41429a2766fccd3bdfbd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/145d67a6e51f463cba7ee3e63174c5d2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/c0f706b9cc0b49e2b556cd0353320230, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/7cf0baa47724416eaa9f729759ec3086, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/911a017fa8b5403f806835a0e5328ad5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/199fbf25b2904851bccdee16a6015db5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/b896f222ab164a2cafeeb5fd72bed93a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e03565e419c44e1eb3c08f915a0be7af, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/42b83905fd894944bb30a421b5736df3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/0578e0a1b97d40bc97ea5a06dabcb251] to archive 2024-11-11T07:31:16,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:31:16,026 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/f185e5e9cf7a4dc492ccdd9c4c0788d6 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/f185e5e9cf7a4dc492ccdd9c4c0788d6 2024-11-11T07:31:16,027 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/787df8e8f6c4472aade64eb76ea5d41d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/787df8e8f6c4472aade64eb76ea5d41d 2024-11-11T07:31:16,028 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/251dd0d4fc7943d7bfaa2c9ff67598bf to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/251dd0d4fc7943d7bfaa2c9ff67598bf 2024-11-11T07:31:16,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/9becb4d43162463ab42b704162a4d30b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/9becb4d43162463ab42b704162a4d30b 2024-11-11T07:31:16,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/76a0b9a88a6a4e9c921d026abdaa34e2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/76a0b9a88a6a4e9c921d026abdaa34e2 2024-11-11T07:31:16,030 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/01699738f6c746919c58a86763a36712 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/01699738f6c746919c58a86763a36712 2024-11-11T07:31:16,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/12e3aa9c7a68499a9dcb8115268a35bb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/12e3aa9c7a68499a9dcb8115268a35bb 2024-11-11T07:31:16,032 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/58cb4f093b7e4dfb82d07ee6b984ce2b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/58cb4f093b7e4dfb82d07ee6b984ce2b 2024-11-11T07:31:16,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/54706287047448919decb07c60f827f2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/54706287047448919decb07c60f827f2 2024-11-11T07:31:16,034 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e9fd86eefde34c97804dbb3a918ae00a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e9fd86eefde34c97804dbb3a918ae00a 2024-11-11T07:31:16,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/a08a80af5bf2433da183d69fbc2285c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/a08a80af5bf2433da183d69fbc2285c8 2024-11-11T07:31:16,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1336199e218b4c929511eeb73ad20a2d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1336199e218b4c929511eeb73ad20a2d 2024-11-11T07:31:16,036 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/b2cf42e07fc149a1b736714c9e2719c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/b2cf42e07fc149a1b736714c9e2719c8 2024-11-11T07:31:16,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/8d9991d938984d43bff72019c0f4ef80 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/8d9991d938984d43bff72019c0f4ef80 2024-11-11T07:31:16,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/3a5d5022ab414540b816d32a126c79a5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/3a5d5022ab414540b816d32a126c79a5 2024-11-11T07:31:16,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/13589f1d1e3f4aa59877cdd03c83d3cc to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/13589f1d1e3f4aa59877cdd03c83d3cc 2024-11-11T07:31:16,040 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1902c68ade564831ae3812d4b2de9395 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1902c68ade564831ae3812d4b2de9395 2024-11-11T07:31:16,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/301e552ddd7f44ed9da3a9d6919fdfdb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/301e552ddd7f44ed9da3a9d6919fdfdb 2024-11-11T07:31:16,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/0d9a92eccfcd4cc9b86d958a8b3451ce to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/0d9a92eccfcd4cc9b86d958a8b3451ce 2024-11-11T07:31:16,042 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/5b3ccae13ee845fea3a7c295b40fc8d1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/5b3ccae13ee845fea3a7c295b40fc8d1 2024-11-11T07:31:16,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/940e59b19620410cbbaaae0f1da8dd9f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/940e59b19620410cbbaaae0f1da8dd9f 2024-11-11T07:31:16,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/a41e6ccf90074d768c592d8bc52fe698 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/a41e6ccf90074d768c592d8bc52fe698 2024-11-11T07:31:16,045 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e417d9b9d73f4a059ca946b0e7baabe1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e417d9b9d73f4a059ca946b0e7baabe1 2024-11-11T07:31:16,051 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/d6f40a8b633b46d6bfe3e5c8496813c1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/d6f40a8b633b46d6bfe3e5c8496813c1 2024-11-11T07:31:16,052 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/617eb9bcdc684ae290970659785dff91 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/617eb9bcdc684ae290970659785dff91 2024-11-11T07:31:16,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e3541e505178446b8132476d874f3e0e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e3541e505178446b8132476d874f3e0e 2024-11-11T07:31:16,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/148f0628487743bca49fbe846c4b27d3 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/148f0628487743bca49fbe846c4b27d3 2024-11-11T07:31:16,055 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/2112ee267acc41429a2766fccd3bdfbd to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/2112ee267acc41429a2766fccd3bdfbd 2024-11-11T07:31:16,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/145d67a6e51f463cba7ee3e63174c5d2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/145d67a6e51f463cba7ee3e63174c5d2 2024-11-11T07:31:16,056 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/c0f706b9cc0b49e2b556cd0353320230 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/c0f706b9cc0b49e2b556cd0353320230 2024-11-11T07:31:16,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/7cf0baa47724416eaa9f729759ec3086 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/7cf0baa47724416eaa9f729759ec3086 2024-11-11T07:31:16,058 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/911a017fa8b5403f806835a0e5328ad5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/911a017fa8b5403f806835a0e5328ad5 2024-11-11T07:31:16,059 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/199fbf25b2904851bccdee16a6015db5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/199fbf25b2904851bccdee16a6015db5 2024-11-11T07:31:16,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/b896f222ab164a2cafeeb5fd72bed93a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/b896f222ab164a2cafeeb5fd72bed93a 2024-11-11T07:31:16,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e03565e419c44e1eb3c08f915a0be7af to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/e03565e419c44e1eb3c08f915a0be7af 2024-11-11T07:31:16,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/42b83905fd894944bb30a421b5736df3 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/42b83905fd894944bb30a421b5736df3 2024-11-11T07:31:16,062 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/0578e0a1b97d40bc97ea5a06dabcb251 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/0578e0a1b97d40bc97ea5a06dabcb251 2024-11-11T07:31:16,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/185bb1db267b4a9c8f624867e958b599, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/bdbc76046b3441af8b6eb267d8f75699, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/e95471e35cd041dfb9a4131edbddd9e2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/92bda31176a140f6b855544bce1915b9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c5a27bc1a8924483b31f4172319a800c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/68acf95de2e84a2f913f4d4d0bbd99c5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/072141a17a99486ea541eb7cd3006309, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/1ae36fdfd62d478b87494ebe758e38ce, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c70ec86a8d42402f80a8276be29adbf0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/8f05f8c594f74d28b88025e794afeaba, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d6be1f5ceeaf493583a9d7c2ccc61dd9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d58c81b2c1dd4fbf851822f476ca061c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c7650788f74c4d07a23db151daede211, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/256b3ecb9f874c12abd3275201cf1e10, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/5b7349723fa8450cbc9e4be2fb40fc7b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/518af87183264c3e891bcb9905f63fa4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/dbddb667df524245b8c6ddf905e01d33, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/b66bb6a854fe4579a04f161d7eb1739a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/23f06fa4ae9d459da59c51002f793efa, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/9c53f8eaf9584b7ba4a2d9e6416f6729, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/16c7e7dab1d14d75845c86c557b803ec, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d724e267965b4c8e8b8016cfeb408581, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/66209a605835431696034626333d9b72, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/37e846bc2d054579a07ebe9231c0d9d4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/1eb97a54c18f4dcb999b536a3f217b8a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/89afc6368deb4f9e8bf4aae26f050630, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/11f4c2ecc27f4573871723fd978e0156, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/13277e4a91c94132bf85c1f8c88cdad4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/7e751ffdd58241908977f210cf4846b6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/55cfca0f0916428988f3f651735bf235, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/040bb5f3b4c64c71b693289ff4c30743, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/b862ce7d46314147a2d2d62c4225eb71, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/a1fe5098b8e540e18b7b1ac4806cbe28, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/a1ae8d4bdb474e98935a37f384cd6b4c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/9db0703b074d400c83e4173cb1180eeb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/88bfe35323454217989f55ceaa2b1d34, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/dbf8dc79c86e4ee7b9e20ed57d690ea1] to archive 2024-11-11T07:31:16,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:31:16,066 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/185bb1db267b4a9c8f624867e958b599 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/185bb1db267b4a9c8f624867e958b599 2024-11-11T07:31:16,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/bdbc76046b3441af8b6eb267d8f75699 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/bdbc76046b3441af8b6eb267d8f75699 2024-11-11T07:31:16,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/e95471e35cd041dfb9a4131edbddd9e2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/e95471e35cd041dfb9a4131edbddd9e2 2024-11-11T07:31:16,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/92bda31176a140f6b855544bce1915b9 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/92bda31176a140f6b855544bce1915b9 2024-11-11T07:31:16,069 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c5a27bc1a8924483b31f4172319a800c to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c5a27bc1a8924483b31f4172319a800c 2024-11-11T07:31:16,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/68acf95de2e84a2f913f4d4d0bbd99c5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/68acf95de2e84a2f913f4d4d0bbd99c5 2024-11-11T07:31:16,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/072141a17a99486ea541eb7cd3006309 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/072141a17a99486ea541eb7cd3006309 2024-11-11T07:31:16,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/1ae36fdfd62d478b87494ebe758e38ce to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/1ae36fdfd62d478b87494ebe758e38ce 2024-11-11T07:31:16,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c70ec86a8d42402f80a8276be29adbf0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c70ec86a8d42402f80a8276be29adbf0 2024-11-11T07:31:16,073 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/8f05f8c594f74d28b88025e794afeaba to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/8f05f8c594f74d28b88025e794afeaba 2024-11-11T07:31:16,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d6be1f5ceeaf493583a9d7c2ccc61dd9 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d6be1f5ceeaf493583a9d7c2ccc61dd9 2024-11-11T07:31:16,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d58c81b2c1dd4fbf851822f476ca061c to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d58c81b2c1dd4fbf851822f476ca061c 2024-11-11T07:31:16,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c7650788f74c4d07a23db151daede211 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/c7650788f74c4d07a23db151daede211 2024-11-11T07:31:16,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/256b3ecb9f874c12abd3275201cf1e10 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/256b3ecb9f874c12abd3275201cf1e10 2024-11-11T07:31:16,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/5b7349723fa8450cbc9e4be2fb40fc7b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/5b7349723fa8450cbc9e4be2fb40fc7b 2024-11-11T07:31:16,078 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/518af87183264c3e891bcb9905f63fa4 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/518af87183264c3e891bcb9905f63fa4 2024-11-11T07:31:16,080 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/dbddb667df524245b8c6ddf905e01d33 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/dbddb667df524245b8c6ddf905e01d33 2024-11-11T07:31:16,089 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/b66bb6a854fe4579a04f161d7eb1739a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/b66bb6a854fe4579a04f161d7eb1739a 2024-11-11T07:31:16,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/23f06fa4ae9d459da59c51002f793efa to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/23f06fa4ae9d459da59c51002f793efa 2024-11-11T07:31:16,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/9c53f8eaf9584b7ba4a2d9e6416f6729 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/9c53f8eaf9584b7ba4a2d9e6416f6729 2024-11-11T07:31:16,092 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/16c7e7dab1d14d75845c86c557b803ec to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/16c7e7dab1d14d75845c86c557b803ec 2024-11-11T07:31:16,093 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d724e267965b4c8e8b8016cfeb408581 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/d724e267965b4c8e8b8016cfeb408581 2024-11-11T07:31:16,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/66209a605835431696034626333d9b72 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/66209a605835431696034626333d9b72 2024-11-11T07:31:16,094 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/37e846bc2d054579a07ebe9231c0d9d4 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/37e846bc2d054579a07ebe9231c0d9d4 2024-11-11T07:31:16,095 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/1eb97a54c18f4dcb999b536a3f217b8a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/1eb97a54c18f4dcb999b536a3f217b8a 2024-11-11T07:31:16,096 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/89afc6368deb4f9e8bf4aae26f050630 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/89afc6368deb4f9e8bf4aae26f050630 2024-11-11T07:31:16,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/11f4c2ecc27f4573871723fd978e0156 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/11f4c2ecc27f4573871723fd978e0156 2024-11-11T07:31:16,097 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/13277e4a91c94132bf85c1f8c88cdad4 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/13277e4a91c94132bf85c1f8c88cdad4 2024-11-11T07:31:16,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/7e751ffdd58241908977f210cf4846b6 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/7e751ffdd58241908977f210cf4846b6 2024-11-11T07:31:16,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/55cfca0f0916428988f3f651735bf235 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/55cfca0f0916428988f3f651735bf235 2024-11-11T07:31:16,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/040bb5f3b4c64c71b693289ff4c30743 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/040bb5f3b4c64c71b693289ff4c30743 2024-11-11T07:31:16,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/b862ce7d46314147a2d2d62c4225eb71 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/b862ce7d46314147a2d2d62c4225eb71 2024-11-11T07:31:16,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/a1fe5098b8e540e18b7b1ac4806cbe28 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/a1fe5098b8e540e18b7b1ac4806cbe28 2024-11-11T07:31:16,102 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/a1ae8d4bdb474e98935a37f384cd6b4c to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/a1ae8d4bdb474e98935a37f384cd6b4c 2024-11-11T07:31:16,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/9db0703b074d400c83e4173cb1180eeb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/9db0703b074d400c83e4173cb1180eeb 2024-11-11T07:31:16,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/88bfe35323454217989f55ceaa2b1d34 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/88bfe35323454217989f55ceaa2b1d34 2024-11-11T07:31:16,104 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/dbf8dc79c86e4ee7b9e20ed57d690ea1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/dbf8dc79c86e4ee7b9e20ed57d690ea1 2024-11-11T07:31:16,106 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f2eafde55fb440fb93195b5fb29d9a17, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/955f8c8847a64786b99b3691ce93b63d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/46e35cd2f9554341bb6279f1b759d28c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/4fcd9448162148b9b113b1b93b794abb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d3486eaa41764214933c8aeb0d243c68, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/66d6ca2be9604bca80b7892607447e14, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/208e24377553410f9aa28d67a4b1b6ae, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f29884bd96be41c29caa2411dc31ea59, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/7d2a0cc163bf4afdadb3be06f72041ed, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/88262fdd99b147f4a335a3c47525c09f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/686058b920bf483f80017f6a6eec2d5b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/98b71eade37e4705a69ae60bbed7b9e9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ae37277256f24c959aece33a8404491f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d12280f48610440b9b93472b0cf02b70, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/9031b144d14c43c6a9853b4788e53246, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/02e859b1cbcb4a499c773fab91b076e1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/8ec20ea326704c11b46ee4f3a9204fd6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/4228dbcafef642659fae83ece981d378, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/41a3e9b99bf84f14a67649a4c0958c81, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ad6b0bb528b74b429702cd6d804bda4f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/139a0aaee16c4792958593123aeebff3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d1d0e5cb6ba84b15bca6abeacc620141, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ad81013a041a468ba867d12c0f349859, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/5ef0fd1a8feb43058a358d416a1e509f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/24eea72a9a6340a78365cd4e309530f4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/cbdd7d8cda5a41c3a39cbfe2f810cfa8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/331119a96eec4d8a99b6528df92d5b14, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/dc1847fcae25409fad97132f001eaaf1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/627edce4cf8f438d8aadeab3de9e9368, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f673aa2a6f4b4e41b337b57dab1033eb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/427c1a5cb44147f39126649d1d3382b2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ee4ac3af77b54fc0849ae8b543308702, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/8b264563401a497387e3a37a576984c7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/dc1d4ae7a03340c7ae966b2ad8df3fd3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/00a4490cfe0b4260a0d00997e0e569ac, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/015289fdb97c4a84900f2031f2b2298f] to archive 2024-11-11T07:31:16,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:31:16,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f2eafde55fb440fb93195b5fb29d9a17 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f2eafde55fb440fb93195b5fb29d9a17 2024-11-11T07:31:16,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/955f8c8847a64786b99b3691ce93b63d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/955f8c8847a64786b99b3691ce93b63d 2024-11-11T07:31:16,110 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/46e35cd2f9554341bb6279f1b759d28c to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/46e35cd2f9554341bb6279f1b759d28c 2024-11-11T07:31:16,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/4fcd9448162148b9b113b1b93b794abb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/4fcd9448162148b9b113b1b93b794abb 2024-11-11T07:31:16,112 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d3486eaa41764214933c8aeb0d243c68 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d3486eaa41764214933c8aeb0d243c68 2024-11-11T07:31:16,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/66d6ca2be9604bca80b7892607447e14 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/66d6ca2be9604bca80b7892607447e14 2024-11-11T07:31:16,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/208e24377553410f9aa28d67a4b1b6ae to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/208e24377553410f9aa28d67a4b1b6ae 2024-11-11T07:31:16,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f29884bd96be41c29caa2411dc31ea59 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f29884bd96be41c29caa2411dc31ea59 2024-11-11T07:31:16,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/7d2a0cc163bf4afdadb3be06f72041ed to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/7d2a0cc163bf4afdadb3be06f72041ed 2024-11-11T07:31:16,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/88262fdd99b147f4a335a3c47525c09f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/88262fdd99b147f4a335a3c47525c09f 2024-11-11T07:31:16,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/686058b920bf483f80017f6a6eec2d5b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/686058b920bf483f80017f6a6eec2d5b 2024-11-11T07:31:16,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/98b71eade37e4705a69ae60bbed7b9e9 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/98b71eade37e4705a69ae60bbed7b9e9 2024-11-11T07:31:16,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ae37277256f24c959aece33a8404491f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ae37277256f24c959aece33a8404491f 2024-11-11T07:31:16,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d12280f48610440b9b93472b0cf02b70 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d12280f48610440b9b93472b0cf02b70 2024-11-11T07:31:16,120 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/9031b144d14c43c6a9853b4788e53246 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/9031b144d14c43c6a9853b4788e53246 2024-11-11T07:31:16,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/02e859b1cbcb4a499c773fab91b076e1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/02e859b1cbcb4a499c773fab91b076e1 2024-11-11T07:31:16,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/8ec20ea326704c11b46ee4f3a9204fd6 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/8ec20ea326704c11b46ee4f3a9204fd6 2024-11-11T07:31:16,122 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/4228dbcafef642659fae83ece981d378 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/4228dbcafef642659fae83ece981d378 2024-11-11T07:31:16,124 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/41a3e9b99bf84f14a67649a4c0958c81 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/41a3e9b99bf84f14a67649a4c0958c81 2024-11-11T07:31:16,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ad6b0bb528b74b429702cd6d804bda4f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ad6b0bb528b74b429702cd6d804bda4f 2024-11-11T07:31:16,125 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/139a0aaee16c4792958593123aeebff3 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/139a0aaee16c4792958593123aeebff3 2024-11-11T07:31:16,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d1d0e5cb6ba84b15bca6abeacc620141 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d1d0e5cb6ba84b15bca6abeacc620141 2024-11-11T07:31:16,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ad81013a041a468ba867d12c0f349859 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ad81013a041a468ba867d12c0f349859 2024-11-11T07:31:16,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/5ef0fd1a8feb43058a358d416a1e509f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/5ef0fd1a8feb43058a358d416a1e509f 2024-11-11T07:31:16,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/24eea72a9a6340a78365cd4e309530f4 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/24eea72a9a6340a78365cd4e309530f4 2024-11-11T07:31:16,129 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/cbdd7d8cda5a41c3a39cbfe2f810cfa8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/cbdd7d8cda5a41c3a39cbfe2f810cfa8 2024-11-11T07:31:16,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/331119a96eec4d8a99b6528df92d5b14 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/331119a96eec4d8a99b6528df92d5b14 2024-11-11T07:31:16,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/dc1847fcae25409fad97132f001eaaf1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/dc1847fcae25409fad97132f001eaaf1 2024-11-11T07:31:16,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/627edce4cf8f438d8aadeab3de9e9368 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/627edce4cf8f438d8aadeab3de9e9368 2024-11-11T07:31:16,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f673aa2a6f4b4e41b337b57dab1033eb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/f673aa2a6f4b4e41b337b57dab1033eb 2024-11-11T07:31:16,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/427c1a5cb44147f39126649d1d3382b2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/427c1a5cb44147f39126649d1d3382b2 2024-11-11T07:31:16,134 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ee4ac3af77b54fc0849ae8b543308702 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/ee4ac3af77b54fc0849ae8b543308702 2024-11-11T07:31:16,135 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/8b264563401a497387e3a37a576984c7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/8b264563401a497387e3a37a576984c7 2024-11-11T07:31:16,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/dc1d4ae7a03340c7ae966b2ad8df3fd3 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/dc1d4ae7a03340c7ae966b2ad8df3fd3 2024-11-11T07:31:16,136 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/00a4490cfe0b4260a0d00997e0e569ac to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/00a4490cfe0b4260a0d00997e0e569ac 2024-11-11T07:31:16,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/015289fdb97c4a84900f2031f2b2298f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/015289fdb97c4a84900f2031f2b2298f 2024-11-11T07:31:16,141 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/recovered.edits/574.seqid, newMaxSeqId=574, maxSeqId=4 2024-11-11T07:31:16,142 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. 2024-11-11T07:31:16,142 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] regionserver.HRegion(1676): Region close journal for 6c7e70f5adf27c26564bc9bc160230a0: Waiting for close lock at 1731310274360Running coprocessor pre-close hooks at 1731310274360Disabling compacts and flushes for region at 1731310274360Disabling writes for close at 1731310274360Obtaining lock to block concurrent updates at 1731310274360Preparing flush snapshotting stores in 6c7e70f5adf27c26564bc9bc160230a0 at 1731310274360Finished memstore snapshotting TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731310274361 (+1 ms)Flushing stores of TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0. at 1731310274361Flushing 6c7e70f5adf27c26564bc9bc160230a0/A: creating writer at 1731310274361Flushing mob file 6c7e70f5adf27c26564bc9bc160230a0/A: appending metadata at 1731310274366 (+5 ms)Flushing mob file 6c7e70f5adf27c26564bc9bc160230a0/A: closing flushed file at 1731310274366Flushing 6c7e70f5adf27c26564bc9bc160230a0/A: appending metadata at 1731310274773 (+407 ms)Flushing 6c7e70f5adf27c26564bc9bc160230a0/A: closing flushed file at 1731310274773Flushing 6c7e70f5adf27c26564bc9bc160230a0/B: creating writer at 1731310275186 (+413 ms)Flushing 6c7e70f5adf27c26564bc9bc160230a0/B: appending metadata at 1731310275188 (+2 ms)Flushing 6c7e70f5adf27c26564bc9bc160230a0/B: closing flushed file at 1731310275188Flushing 6c7e70f5adf27c26564bc9bc160230a0/C: creating writer at 1731310275598 (+410 ms)Flushing 6c7e70f5adf27c26564bc9bc160230a0/C: appending metadata at 1731310275600 (+2 ms)Flushing 6c7e70f5adf27c26564bc9bc160230a0/C: closing flushed file at 1731310275600Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f0bb1fb: reopening flushed file at 1731310276011 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e7e25f7: reopening flushed file at 1731310276015 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71c014b9: reopening flushed file at 1731310276019 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 6c7e70f5adf27c26564bc9bc160230a0 in 1663ms, sequenceid=571, compaction requested=true at 1731310276023 (+4 ms)Writing region close event to WAL at 1731310276139 (+116 ms)Running coprocessor post-close hooks at 1731310276142 (+3 ms)Closed at 1731310276142 2024-11-11T07:31:16,144 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=120}] handler.UnassignRegionHandler(157): Closed 6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,144 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=6c7e70f5adf27c26564bc9bc160230a0, regionState=CLOSED 2024-11-11T07:31:16,145 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:31:16,147 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-11T07:31:16,147 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; CloseRegionProcedure 6c7e70f5adf27c26564bc9bc160230a0, server=ce2cf0244921,33345,1731310155369 in 1.9380 sec 2024-11-11T07:31:16,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-11T07:31:16,148 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6c7e70f5adf27c26564bc9bc160230a0, UNASSIGN in 1.9420 sec 2024-11-11T07:31:16,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-11T07:31:16,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9450 sec 2024-11-11T07:31:16,152 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310276152"}]},"ts":"1731310276152"} 2024-11-11T07:31:16,153 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T07:31:16,153 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T07:31:16,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9540 sec 2024-11-11T07:31:16,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-11T07:31:16,345 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:16,345 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T07:31:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:16,347 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=121, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:16,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T07:31:16,348 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=121, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:16,349 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,351 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/recovered.edits] 2024-11-11T07:31:16,353 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1db0a7bce92c4eee9fcfdb4917c0d49a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/1db0a7bce92c4eee9fcfdb4917c0d49a 2024-11-11T07:31:16,354 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/c2a05e412581497da7809c2305578c9f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/c2a05e412581497da7809c2305578c9f 2024-11-11T07:31:16,355 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/f0dddb0c8d5649b2a1441920979c62aa to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/A/f0dddb0c8d5649b2a1441920979c62aa 2024-11-11T07:31:16,357 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/b9783ca4228c42a2844ed866c6faf3e6 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/b9783ca4228c42a2844ed866c6faf3e6 2024-11-11T07:31:16,358 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/ef007f379d3f4739abd640134f34e1b9 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/ef007f379d3f4739abd640134f34e1b9 2024-11-11T07:31:16,359 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/f07c8dd2c4da4eeaa5e6213aa9d02f70 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/B/f07c8dd2c4da4eeaa5e6213aa9d02f70 2024-11-11T07:31:16,361 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/16dbb1501c59485daa3a50d6caf613f2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/16dbb1501c59485daa3a50d6caf613f2 2024-11-11T07:31:16,362 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/555c3811855343fd9f774a87addd4fb7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/555c3811855343fd9f774a87addd4fb7 2024-11-11T07:31:16,363 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d4486bb0a68e4e28b9614225c880f869 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/C/d4486bb0a68e4e28b9614225c880f869 2024-11-11T07:31:16,365 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/recovered.edits/574.seqid to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0/recovered.edits/574.seqid 2024-11-11T07:31:16,365 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,365 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T07:31:16,366 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T07:31:16,366 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T07:31:16,369 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111106b07c29d35646e8b2361fe9f4ecd227_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111106b07c29d35646e8b2361fe9f4ecd227_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,370 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110985170864724bf685cdd61a5d7cb7b8_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110985170864724bf685cdd61a5d7cb7b8_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,371 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111164c6c222d684846aaacd74b3805bbd7_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111164c6c222d684846aaacd74b3805bbd7_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,372 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111221649433cc846a79a4419a3e8224871_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111221649433cc846a79a4419a3e8224871_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,372 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111146a97ce439224912966bdf48619bb18c_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111146a97ce439224912966bdf48619bb18c_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,373 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114b6ebb3981e542fe86c3c0d81d3922bf_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114b6ebb3981e542fe86c3c0d81d3922bf_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,374 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114c46567e50364617b3fc7fa9d9b29ccb_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114c46567e50364617b3fc7fa9d9b29ccb_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,375 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115d306f8893d944a9b7c1b8abd17a9c3a_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411115d306f8893d944a9b7c1b8abd17a9c3a_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,375 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111623f889444dd4fd69dab30f07a08e73e_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111623f889444dd4fd69dab30f07a08e73e_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,376 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111170863f9697ca46c5822de1ee1b9397cf_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111170863f9697ca46c5822de1ee1b9397cf_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,377 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111830a1aec1670418a80e3191e7ad0c972_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111830a1aec1670418a80e3191e7ad0c972_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,378 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111885d917171054354a42bd39375c1f620_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111885d917171054354a42bd39375c1f620_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,378 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118bac7ef083ed4c8babbefea027495841_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118bac7ef083ed4c8babbefea027495841_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,379 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111922077b91cb74e5d9bbed5ae93ae5e4f_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111922077b91cb74e5d9bbed5ae93ae5e4f_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,380 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111929b132f4cc44a8fba2041ff25427380_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111929b132f4cc44a8fba2041ff25427380_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,381 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111195edbc4ffa1147c383674723044e8260_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111195edbc4ffa1147c383674723044e8260_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,382 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119b3bd02078624384adb7934c24d424cb_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119b3bd02078624384adb7934c24d424cb_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,382 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a01b389728634abbbb709d22c80ddb0d_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a01b389728634abbbb709d22c80ddb0d_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,383 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a3138de334504ed784496b5c82426da1_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a3138de334504ed784496b5c82426da1_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,384 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a84ec0a34def4611b5f1550e52848d6a_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111a84ec0a34def4611b5f1550e52848d6a_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,385 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ae8b84e4efbd4f019aa7b086a90ef22d_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111ae8b84e4efbd4f019aa7b086a90ef22d_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,385 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b4e0abe6d80048808f25100cc82e1477_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b4e0abe6d80048808f25100cc82e1477_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,386 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b7b0df26a4d1484b817ff194d1948808_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b7b0df26a4d1484b817ff194d1948808_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,387 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b982a20e73c04b96b7b6ab0d71f2d6ad_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b982a20e73c04b96b7b6ab0d71f2d6ad_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,388 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5797afd66c74bd883dfddb25f2dbc50_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d5797afd66c74bd883dfddb25f2dbc50_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,388 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d804a3855d06471eb4401245aa13a570_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d804a3855d06471eb4401245aa13a570_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,389 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d9b67a8bb3774c4abc58e7f8d0f0918f_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d9b67a8bb3774c4abc58e7f8d0f0918f_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,390 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111df6e89b782c14865a4cc81e14245f366_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111df6e89b782c14865a4cc81e14245f366_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,391 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e99b6376e3b44c8480fc46b85407d164_6c7e70f5adf27c26564bc9bc160230a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111e99b6376e3b44c8480fc46b85407d164_6c7e70f5adf27c26564bc9bc160230a0 2024-11-11T07:31:16,392 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T07:31:16,394 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=121, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:16,397 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T07:31:16,399 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T07:31:16,400 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=121, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:16,400 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T07:31:16,400 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731310276400"}]},"ts":"9223372036854775807"} 2024-11-11T07:31:16,402 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T07:31:16,402 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6c7e70f5adf27c26564bc9bc160230a0, NAME => 'TestAcidGuarantees,,1731310240687.6c7e70f5adf27c26564bc9bc160230a0.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T07:31:16,402 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T07:31:16,403 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731310276402"}]},"ts":"9223372036854775807"} 2024-11-11T07:31:16,404 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T07:31:16,405 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=121, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:16,406 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 60 msec 2024-11-11T07:31:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-11T07:31:16,455 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T07:31:16,455 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:16,465 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=247 (was 250), OpenFileDescriptor=449 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=884 (was 924), ProcessCount=11 (was 11), AvailableMemoryMB=2436 (was 2498) 2024-11-11T07:31:16,475 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=247, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=884, ProcessCount=11, AvailableMemoryMB=2435 2024-11-11T07:31:16,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:31:16,477 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:31:16,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:16,478 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T07:31:16,478 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:16,478 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 122 2024-11-11T07:31:16,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-11T07:31:16,479 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T07:31:16,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742336_1512 (size=1061) 2024-11-11T07:31:16,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-11T07:31:16,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-11T07:31:16,890 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e 2024-11-11T07:31:16,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742337_1513 (size=53) 2024-11-11T07:31:17,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-11T07:31:17,295 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:31:17,296 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 752742eb661c603f835a96396e7673f2, disabling compactions & flushes 2024-11-11T07:31:17,296 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:17,296 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:17,296 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. after waiting 0 ms 2024-11-11T07:31:17,296 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:17,296 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:17,296 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 752742eb661c603f835a96396e7673f2: Waiting for close lock at 1731310277296Disabling compacts and flushes for region at 1731310277296Disabling writes for close at 1731310277296Writing region close event to WAL at 1731310277296Closed at 1731310277296 2024-11-11T07:31:17,297 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T07:31:17,297 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731310277297"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731310277297"}]},"ts":"1731310277297"} 2024-11-11T07:31:17,298 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T07:31:17,299 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T07:31:17,299 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310277299"}]},"ts":"1731310277299"} 2024-11-11T07:31:17,300 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T07:31:17,300 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=752742eb661c603f835a96396e7673f2, ASSIGN}] 2024-11-11T07:31:17,301 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=752742eb661c603f835a96396e7673f2, ASSIGN 2024-11-11T07:31:17,302 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=752742eb661c603f835a96396e7673f2, ASSIGN; state=OFFLINE, location=ce2cf0244921,33345,1731310155369; forceNewPlan=false, retain=false 2024-11-11T07:31:17,453 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=752742eb661c603f835a96396e7673f2, regionState=OPENING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:17,454 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=123, ppid=122, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=752742eb661c603f835a96396e7673f2, ASSIGN because future has completed 2024-11-11T07:31:17,454 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:31:17,609 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:17,609 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7752): Opening region: {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:31:17,610 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:17,610 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:31:17,610 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7794): checking encryption for 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:17,610 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(7797): checking classloading for 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:17,611 INFO [StoreOpener-752742eb661c603f835a96396e7673f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:17,612 INFO [StoreOpener-752742eb661c603f835a96396e7673f2-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:31:17,612 INFO [StoreOpener-752742eb661c603f835a96396e7673f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 752742eb661c603f835a96396e7673f2 columnFamilyName A 2024-11-11T07:31:17,612 DEBUG [StoreOpener-752742eb661c603f835a96396e7673f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:17,612 INFO [StoreOpener-752742eb661c603f835a96396e7673f2-1 {}] regionserver.HStore(327): Store=752742eb661c603f835a96396e7673f2/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:31:17,612 INFO [StoreOpener-752742eb661c603f835a96396e7673f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:17,613 INFO [StoreOpener-752742eb661c603f835a96396e7673f2-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:31:17,613 INFO [StoreOpener-752742eb661c603f835a96396e7673f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 752742eb661c603f835a96396e7673f2 columnFamilyName B 2024-11-11T07:31:17,613 DEBUG [StoreOpener-752742eb661c603f835a96396e7673f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:17,614 INFO [StoreOpener-752742eb661c603f835a96396e7673f2-1 {}] regionserver.HStore(327): Store=752742eb661c603f835a96396e7673f2/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:31:17,614 INFO [StoreOpener-752742eb661c603f835a96396e7673f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:17,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-11T07:31:17,615 INFO [StoreOpener-752742eb661c603f835a96396e7673f2-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:31:17,615 INFO [StoreOpener-752742eb661c603f835a96396e7673f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 752742eb661c603f835a96396e7673f2 columnFamilyName C 2024-11-11T07:31:17,615 DEBUG [StoreOpener-752742eb661c603f835a96396e7673f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:17,615 INFO [StoreOpener-752742eb661c603f835a96396e7673f2-1 {}] regionserver.HStore(327): Store=752742eb661c603f835a96396e7673f2/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:31:17,615 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:17,615 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1038): replaying wal for 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:17,616 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2 2024-11-11T07:31:17,616 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2 2024-11-11T07:31:17,617 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1048): stopping wal replay for 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:17,617 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1060): Cleaning up temporary data for 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:17,617 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:31:17,618 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1093): writing seq id for 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:17,620 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T07:31:17,620 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1114): Opened 752742eb661c603f835a96396e7673f2; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60370653, jitterRate=-0.10040716826915741}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:31:17,620 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:17,621 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegion(1006): Region open journal for 752742eb661c603f835a96396e7673f2: Running coprocessor pre-open hook at 1731310277610Writing region info on filesystem at 1731310277610Initializing all the Stores at 1731310277610Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310277610Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310277611 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310277611Cleaning up temporary data from old regions at 1731310277617 (+6 ms)Running coprocessor post-open hooks at 1731310277620 (+3 ms)Region opened successfully at 1731310277621 (+1 ms) 2024-11-11T07:31:17,622 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., pid=124, masterSystemTime=1731310277606 2024-11-11T07:31:17,623 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:17,623 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=124}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:17,624 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=123 updating hbase:meta row=752742eb661c603f835a96396e7673f2, regionState=OPEN, openSeqNum=2, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:17,625 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=124, ppid=123, state=RUNNABLE, hasLock=false; OpenRegionProcedure 752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:31:17,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-11T07:31:17,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; OpenRegionProcedure 752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 in 172 msec 2024-11-11T07:31:17,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=123, resume processing ppid=122 2024-11-11T07:31:17,629 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, ppid=122, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=752742eb661c603f835a96396e7673f2, ASSIGN in 327 msec 2024-11-11T07:31:17,630 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T07:31:17,630 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310277630"}]},"ts":"1731310277630"} 2024-11-11T07:31:17,631 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T07:31:17,632 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=122, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T07:31:17,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1550 sec 2024-11-11T07:31:17,943 WARN [regionserver/ce2cf0244921:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-11T07:31:18,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=122 2024-11-11T07:31:18,624 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:18,625 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@204ac613, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,626 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:18,626 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:18,627 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53bbffe0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:18,627 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:18,628 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45482, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45c203ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,629 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:18,630 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:18,631 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:18,631 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T07:31:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is ce2cf0244921,33029,1731310154427 2024-11-11T07:31:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@60fb6765 2024-11-11T07:31:18,632 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T07:31:18,633 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45486, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T07:31:18,634 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74bdbad8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,634 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:18,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:18,635 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:18,635 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3253c8d9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:18,636 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:18,637 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:18,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b1e5813, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,638 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ac69b7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,638 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:18,638 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:18,639 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:18,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:18,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:18,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d7a940c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:18,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:18,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:18,640 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b2a4e88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@277a5c68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:18,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:18,642 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b37643, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:18,643 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:18,644 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@314e9ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,644 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1df5cdc7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,645 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:18,645 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:18,645 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:18,645 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7df90283, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:18,646 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:18,647 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:18,647 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a7c4f35, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,648 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22b9b902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,648 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:18,648 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:18,649 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ddc816f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:18,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:18,650 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:18,650 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@503eea90, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,651 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9f4da0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,651 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:18,651 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:18,652 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39bcb282, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:18,652 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:18,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:18,654 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:18,655 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a03ce88, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,655 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@107467c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,655 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:18,655 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:18,656 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:18,656 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f0ee36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:18,657 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:18,658 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45606, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:18,658 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@177fb7c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,659 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2acfd5e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,659 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:18,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:18,661 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fe7567d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:18,662 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61b223b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,663 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@379767c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,663 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:18,663 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:18,664 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30b94e0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:18,665 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:18,666 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45632, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:18,667 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c312443, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,668 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@69f15306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,668 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:18,668 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:18,668 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26ca972f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:18,669 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:18,670 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:18,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@769f1099, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:18,671 DEBUG [Thread-2271 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:18,671 DEBUG [Thread-2282 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:18,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:18,673 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:18,675 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:31:18,676 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:31:18,687 DEBUG [Thread-2275 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:18,687 DEBUG [Thread-2286 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:18,687 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:18,688 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:18,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36244, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:18,689 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:31:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:31:18,711 DEBUG [Thread-2284 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:18,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:18,713 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:18,714 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36254, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:18,715 DEBUG [Thread-2269 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:18,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:31:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:18,717 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:18,720 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:18,721 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:31:18,723 DEBUG [Thread-2267 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:18,727 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:18,727 DEBUG [Thread-2273 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:18,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:18,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:31:18,731 DEBUG [Thread-2278 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:18,731 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:18,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-11T07:31:18,732 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:18,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T07:31:18,733 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:18,735 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:18,735 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:31:18,736 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:18,736 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:18,736 DEBUG [Thread-2280 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:18,737 DEBUG [Thread-2280 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:18,737 DEBUG [Thread-2280 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:18,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:31:18,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:18,742 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T07:31:18,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:18,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:18,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:18,743 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:18,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:18,744 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:18,759 DEBUG [Thread-2273 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:18,760 DEBUG [Thread-2273 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:18,761 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36304, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:18,762 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2] 2024-11-11T07:31:18,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/a8c74bd1523a47a486f4be4a2b574692 is 50, key is test_row_0/A:col10/1731310278741/Put/seqid=0 2024-11-11T07:31:18,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T07:31:18,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742338_1514 (size=19021) 2024-11-11T07:31:18,858 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/a8c74bd1523a47a486f4be4a2b574692 2024-11-11T07:31:18,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310278791, max timestamp=1731310278855Num uniques 859; 2024-11-11T07:31:18,860 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:31:18,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310278791, max timestamp=1731310278855Num uniques 859; 2024-11-11T07:31:18,861 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:31:18,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310278791, max timestamp=1731310278855Num uniques 859; 2024-11-11T07:31:18,863 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:31:18,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/eb15f13dc16e4a65ace88887f5e24aac is 50, key is test_row_0/B:col10/1731310278741/Put/seqid=0 2024-11-11T07:31:18,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:31:18,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:18,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:18,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:18,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:18,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:18,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:18,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742339_1515 (size=12001) 2024-11-11T07:31:18,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/eb15f13dc16e4a65ace88887f5e24aac 2024-11-11T07:31:18,937 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/20bd7bf2e7f143fc817e9ba7b81ebd4d is 50, key is test_row_0/C:col10/1731310278741/Put/seqid=0 2024-11-11T07:31:18,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:18,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310338914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:18,940 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:18,941 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:18,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742340_1516 (size=12001) 2024-11-11T07:31:18,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/20bd7bf2e7f143fc817e9ba7b81ebd4d 2024-11-11T07:31:18,946 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/a8c74bd1523a47a486f4be4a2b574692 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/a8c74bd1523a47a486f4be4a2b574692 2024-11-11T07:31:18,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310338933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:18,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310338934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:18,949 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:18,953 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:18,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310338936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:18,953 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/a8c74bd1523a47a486f4be4a2b574692, entries=300, sequenceid=12, filesize=18.6 K 2024-11-11T07:31:18,956 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:18,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310338940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:18,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:18,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:18,956 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:18,959 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/eb15f13dc16e4a65ace88887f5e24aac as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/eb15f13dc16e4a65ace88887f5e24aac 2024-11-11T07:31:18,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/eb15f13dc16e4a65ace88887f5e24aac, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T07:31:18,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/20bd7bf2e7f143fc817e9ba7b81ebd4d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/20bd7bf2e7f143fc817e9ba7b81ebd4d 2024-11-11T07:31:18,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/20bd7bf2e7f143fc817e9ba7b81ebd4d, entries=150, sequenceid=12, filesize=11.7 K 2024-11-11T07:31:18,975 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 752742eb661c603f835a96396e7673f2 in 233ms, sequenceid=12, compaction requested=false 2024-11-11T07:31:18,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:19,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-11T07:31:19,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:19,052 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-11T07:31:19,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:19,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:19,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:19,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:19,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:19,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:19,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T07:31:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:19,061 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:19,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/c3f5ebb9c8894ea7a474ef5e28097c0a is 50, key is test_row_0/A:col10/1731310278911/Put/seqid=0 2024-11-11T07:31:19,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742341_1517 (size=12001) 2024-11-11T07:31:19,094 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310339078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,095 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310339080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310339081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310339082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310339095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,111 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,216 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310339205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,219 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310339215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310339215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310339215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310339216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T07:31:19,435 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310339424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,436 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310339435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310339436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310339436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310339436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,440 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,492 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/c3f5ebb9c8894ea7a474ef5e28097c0a 2024-11-11T07:31:19,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/da7432f9a07d48be99bcc55e77373834 is 50, key is test_row_0/B:col10/1731310278911/Put/seqid=0 2024-11-11T07:31:19,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742342_1518 (size=12001) 2024-11-11T07:31:19,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310339746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310339746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,758 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310339746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310339747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,760 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:19,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310339747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:19,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:19,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T07:31:19,913 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/da7432f9a07d48be99bcc55e77373834 2024-11-11T07:31:19,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/16c334de86b546dd87dee814c6150891 is 50, key is test_row_0/C:col10/1731310278911/Put/seqid=0 2024-11-11T07:31:19,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742343_1519 (size=12001) 2024-11-11T07:31:20,274 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310340266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:20,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:20,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310340266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310340266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:20,275 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310340266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:20,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:20,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310340275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:20,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:20,395 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/16c334de86b546dd87dee814c6150891 2024-11-11T07:31:20,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/c3f5ebb9c8894ea7a474ef5e28097c0a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c3f5ebb9c8894ea7a474ef5e28097c0a 2024-11-11T07:31:20,419 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c3f5ebb9c8894ea7a474ef5e28097c0a, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T07:31:20,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/da7432f9a07d48be99bcc55e77373834 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/da7432f9a07d48be99bcc55e77373834 2024-11-11T07:31:20,424 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/da7432f9a07d48be99bcc55e77373834, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T07:31:20,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/16c334de86b546dd87dee814c6150891 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/16c334de86b546dd87dee814c6150891 2024-11-11T07:31:20,429 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/16c334de86b546dd87dee814c6150891, entries=150, sequenceid=48, filesize=11.7 K 2024-11-11T07:31:20,431 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 752742eb661c603f835a96396e7673f2 in 1379ms, sequenceid=48, compaction requested=false 2024-11-11T07:31:20,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:20,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:20,431 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-11T07:31:20,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-11T07:31:20,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-11T07:31:20,445 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6970 sec 2024-11-11T07:31:20,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 1.7140 sec 2024-11-11T07:31:20,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-11T07:31:20,898 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:20,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:20,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-11T07:31:20,902 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:20,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T07:31:20,903 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:20,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:21,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T07:31:21,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-11T07:31:21,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:21,056 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T07:31:21,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:21,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:21,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:21,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:21,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:21,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:21,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/1c25d5e8aab34d559fbec171823f349a is 50, key is test_row_0/A:col10/1731310279091/Put/seqid=0 2024-11-11T07:31:21,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742344_1520 (size=12001) 2024-11-11T07:31:21,084 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/1c25d5e8aab34d559fbec171823f349a 2024-11-11T07:31:21,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/f91c27309c744c4cb47043392c3454bf is 50, key is test_row_0/B:col10/1731310279091/Put/seqid=0 2024-11-11T07:31:21,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742345_1521 (size=12001) 2024-11-11T07:31:21,111 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/f91c27309c744c4cb47043392c3454bf 2024-11-11T07:31:21,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/ad52b78c19854fada904a2d1d7a6a3c8 is 50, key is test_row_0/C:col10/1731310279091/Put/seqid=0 2024-11-11T07:31:21,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742346_1522 (size=12001) 2024-11-11T07:31:21,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T07:31:21,290 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:21,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9554, totalHeapSize=21976, min timestamp=1731310281325, max timestamp=1731310281367Num uniques 859; 2024-11-11T07:31:21,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=9554, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T07:31:21,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731310281325, max timestamp=1731310281367Num uniques 859; 2024-11-11T07:31:21,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731310281325, max timestamp=1731310281367Num uniques 859; 2024-11-11T07:31:21,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T07:31:21,390 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39324, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T07:31:21,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310341330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310341459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,462 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310341462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310341462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310341462, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,483 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,484 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310341505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T07:31:21,541 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/ad52b78c19854fada904a2d1d7a6a3c8 2024-11-11T07:31:21,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/1c25d5e8aab34d559fbec171823f349a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1c25d5e8aab34d559fbec171823f349a 2024-11-11T07:31:21,555 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1c25d5e8aab34d559fbec171823f349a, entries=150, sequenceid=59, filesize=11.7 K 2024-11-11T07:31:21,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/f91c27309c744c4cb47043392c3454bf as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f91c27309c744c4cb47043392c3454bf 2024-11-11T07:31:21,560 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f91c27309c744c4cb47043392c3454bf, entries=150, sequenceid=59, filesize=11.7 K 2024-11-11T07:31:21,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/ad52b78c19854fada904a2d1d7a6a3c8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ad52b78c19854fada904a2d1d7a6a3c8 2024-11-11T07:31:21,568 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ad52b78c19854fada904a2d1d7a6a3c8, entries=150, sequenceid=59, filesize=11.7 K 2024-11-11T07:31:21,569 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=128.94 KB/132032 for 752742eb661c603f835a96396e7673f2 in 513ms, sequenceid=59, compaction requested=true 2024-11-11T07:31:21,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:21,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:21,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-11T07:31:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-11T07:31:21,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-11T07:31:21,574 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 669 msec 2024-11-11T07:31:21,575 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 675 msec 2024-11-11T07:31:21,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:21,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=135.65 KB heapSize=388.23 KB 2024-11-11T07:31:21,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:21,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:21,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:21,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:21,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:21,580 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:21,589 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/82338252e26a4540aa166cdad1ec6cc7 is 50, key is test_row_0/A:col10/1731310281456/Put/seqid=0 2024-11-11T07:31:21,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742347_1523 (size=14341) 2024-11-11T07:31:21,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310341614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310341614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310341620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,635 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310341622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,719 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T07:31:21,734 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310341726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,734 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,735 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310341745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,759 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310341745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,760 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,760 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310341745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,761 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310341755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310341964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310341964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,981 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310341964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:21,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310341978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:21,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:21,998 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:22,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/82338252e26a4540aa166cdad1ec6cc7 2024-11-11T07:31:22,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/f19d578d50ee4c8183b77f801e5a497e is 50, key is test_row_0/B:col10/1731310281456/Put/seqid=0 2024-11-11T07:31:22,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-11T07:31:22,045 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:22,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-11T07:31:22,048 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:22,049 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:22,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:22,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T07:31:22,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:22,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310342045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:22,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,059 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:22,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742348_1524 (size=12001) 2024-11-11T07:31:22,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T07:31:22,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T07:31:22,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:22,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:22,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:22,204 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310342286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:22,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:22,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310342286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,289 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:22,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:22,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310342295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:22,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:22,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310342305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:22,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,321 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:22,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T07:31:22,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:22,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:22,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:22,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T07:31:22,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.34 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/f19d578d50ee4c8183b77f801e5a497e 2024-11-11T07:31:22,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/c4f3dc29fb00470f9b3ce48450245947 is 50, key is test_row_0/C:col10/1731310281456/Put/seqid=0 2024-11-11T07:31:22,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742349_1525 (size=12001) 2024-11-11T07:31:22,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.34 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/c4f3dc29fb00470f9b3ce48450245947 2024-11-11T07:31:22,498 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/82338252e26a4540aa166cdad1ec6cc7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/82338252e26a4540aa166cdad1ec6cc7 2024-11-11T07:31:22,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/82338252e26a4540aa166cdad1ec6cc7, entries=200, sequenceid=96, filesize=14.0 K 2024-11-11T07:31:22,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/f19d578d50ee4c8183b77f801e5a497e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f19d578d50ee4c8183b77f801e5a497e 2024-11-11T07:31:22,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T07:31:22,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:22,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:22,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:22,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f19d578d50ee4c8183b77f801e5a497e, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T07:31:22,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/c4f3dc29fb00470f9b3ce48450245947 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/c4f3dc29fb00470f9b3ce48450245947 2024-11-11T07:31:22,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/c4f3dc29fb00470f9b3ce48450245947, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T07:31:22,535 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~135.65 KB/138902, heapSize ~388.19 KB/397504, currentSize=46.96 KB/48090 for 752742eb661c603f835a96396e7673f2 in 956ms, sequenceid=96, compaction requested=true 2024-11-11T07:31:22,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:22,535 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:22,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:22,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:22,536 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:22,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:22,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:22,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:22,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:22,538 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57364 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:22,538 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/A is initiating minor compaction (all files) 2024-11-11T07:31:22,538 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/A in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:22,538 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/a8c74bd1523a47a486f4be4a2b574692, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c3f5ebb9c8894ea7a474ef5e28097c0a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1c25d5e8aab34d559fbec171823f349a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/82338252e26a4540aa166cdad1ec6cc7] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=56.0 K 2024-11-11T07:31:22,538 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8c74bd1523a47a486f4be4a2b574692, keycount=300, bloomtype=ROW, size=18.6 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731310278681 2024-11-11T07:31:22,539 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:22,539 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/B is initiating minor compaction (all files) 2024-11-11T07:31:22,539 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/B in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:22,539 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/eb15f13dc16e4a65ace88887f5e24aac, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/da7432f9a07d48be99bcc55e77373834, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f91c27309c744c4cb47043392c3454bf, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f19d578d50ee4c8183b77f801e5a497e] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=46.9 K 2024-11-11T07:31:22,539 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3f5ebb9c8894ea7a474ef5e28097c0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731310278911 2024-11-11T07:31:22,539 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting eb15f13dc16e4a65ace88887f5e24aac, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731310278692 2024-11-11T07:31:22,539 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c25d5e8aab34d559fbec171823f349a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731310279071 2024-11-11T07:31:22,540 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting da7432f9a07d48be99bcc55e77373834, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731310278911 2024-11-11T07:31:22,540 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82338252e26a4540aa166cdad1ec6cc7, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731310281455 2024-11-11T07:31:22,540 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f91c27309c744c4cb47043392c3454bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731310279071 2024-11-11T07:31:22,541 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f19d578d50ee4c8183b77f801e5a497e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731310281456 2024-11-11T07:31:22,562 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#A#compaction#439 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:22,563 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/508a6fde64754edd85203947da483c75 is 50, key is test_row_0/A:col10/1731310281456/Put/seqid=0 2024-11-11T07:31:22,569 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#B#compaction#440 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:22,570 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/fe9c1db1950f428fb237c9889253c5c7 is 50, key is test_row_0/B:col10/1731310281456/Put/seqid=0 2024-11-11T07:31:22,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:22,579 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T07:31:22,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:22,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:22,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:22,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:22,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:22,579 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:22,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742350_1526 (size=12139) 2024-11-11T07:31:22,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/802c3310b71541198a6e4f2e5d0a3bca is 50, key is test_row_0/A:col10/1731310281610/Put/seqid=0 2024-11-11T07:31:22,613 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/508a6fde64754edd85203947da483c75 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/508a6fde64754edd85203947da483c75 2024-11-11T07:31:22,617 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 752742eb661c603f835a96396e7673f2/A of 752742eb661c603f835a96396e7673f2 into 508a6fde64754edd85203947da483c75(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:22,617 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:22,617 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/A, priority=12, startTime=1731310282535; duration=0sec 2024-11-11T07:31:22,617 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:22,617 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:22,617 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:22,619 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:22,619 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/C is initiating minor compaction (all files) 2024-11-11T07:31:22,620 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/C in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:22,620 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/20bd7bf2e7f143fc817e9ba7b81ebd4d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/16c334de86b546dd87dee814c6150891, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ad52b78c19854fada904a2d1d7a6a3c8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/c4f3dc29fb00470f9b3ce48450245947] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=46.9 K 2024-11-11T07:31:22,620 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20bd7bf2e7f143fc817e9ba7b81ebd4d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731310278692 2024-11-11T07:31:22,620 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16c334de86b546dd87dee814c6150891, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731310278911 2024-11-11T07:31:22,620 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting ad52b78c19854fada904a2d1d7a6a3c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731310279071 2024-11-11T07:31:22,621 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4f3dc29fb00470f9b3ce48450245947, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731310281456 2024-11-11T07:31:22,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742351_1527 (size=12139) 2024-11-11T07:31:22,655 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/fe9c1db1950f428fb237c9889253c5c7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/fe9c1db1950f428fb237c9889253c5c7 2024-11-11T07:31:22,659 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 752742eb661c603f835a96396e7673f2/B of 752742eb661c603f835a96396e7673f2 into fe9c1db1950f428fb237c9889253c5c7(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:22,659 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:22,659 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/B, priority=12, startTime=1731310282536; duration=0sec 2024-11-11T07:31:22,659 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:22,659 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:B 2024-11-11T07:31:22,660 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#C#compaction#442 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:22,661 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/aeb1f3ab761446ba9994a6a6e7cf8af3 is 50, key is test_row_0/C:col10/1731310281456/Put/seqid=0 2024-11-11T07:31:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T07:31:22,679 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T07:31:22,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:22,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:22,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:22,679 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,679 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742353_1529 (size=12139) 2024-11-11T07:31:22,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742352_1528 (size=14341) 2024-11-11T07:31:22,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/802c3310b71541198a6e4f2e5d0a3bca 2024-11-11T07:31:22,720 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/aeb1f3ab761446ba9994a6a6e7cf8af3 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/aeb1f3ab761446ba9994a6a6e7cf8af3 2024-11-11T07:31:22,724 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 752742eb661c603f835a96396e7673f2/C of 752742eb661c603f835a96396e7673f2 into aeb1f3ab761446ba9994a6a6e7cf8af3(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:22,724 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:22,725 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/C, priority=12, startTime=1731310282536; duration=0sec 2024-11-11T07:31:22,725 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:22,725 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:C 2024-11-11T07:31:22,733 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/a097971806514ce7b7af81cd7db44668 is 50, key is test_row_0/B:col10/1731310281610/Put/seqid=0 2024-11-11T07:31:22,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742354_1530 (size=12001) 2024-11-11T07:31:22,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731310282775, max timestamp=1731310282814Num uniques 859; 2024-11-11T07:31:22,818 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T07:31:22,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731310282775, max timestamp=1731310282814Num uniques 859; 2024-11-11T07:31:22,820 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T07:31:22,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731310282775, max timestamp=1731310282814Num uniques 859; 2024-11-11T07:31:22,821 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T07:31:22,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T07:31:22,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:22,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:22,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:22,833 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:22,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310342852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,873 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:22,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:22,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310342859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:22,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310342861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:22,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310342863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:22,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:22,892 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:22,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310342873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,893 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:22,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T07:31:22,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:22,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:22,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:22,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:22,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:22,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310342984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:22,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310342985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:22,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:22,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310342985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:22,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:22,999 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:22,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:22,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310342985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,001 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:23,002 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:23,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310342994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:23,145 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T07:31:23,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:23,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:23,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:23,145 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:23,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:23,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:23,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/a097971806514ce7b7af81cd7db44668 2024-11-11T07:31:23,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T07:31:23,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/933c77e550514821984a598e08848d77 is 50, key is test_row_0/C:col10/1731310281610/Put/seqid=0 2024-11-11T07:31:23,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742355_1531 (size=12001) 2024-11-11T07:31:23,217 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:23,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310343205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:23,217 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:23,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310343205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,218 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:23,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:23,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310343206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:23,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310343206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:23,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310343215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,220 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:23,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T07:31:23,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:23,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:23,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:23,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:23,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:23,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:23,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T07:31:23,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:23,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:23,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:23,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:23,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:23,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:23,533 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310343526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:23,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,534 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:23,534 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:23,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310343526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:23,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,535 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:23,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:23,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310343526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:23,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,535 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:23,535 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:23,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310343527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:23,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:23,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:23,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310343531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:23,546 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:23,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:23,602 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/933c77e550514821984a598e08848d77 2024-11-11T07:31:23,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/802c3310b71541198a6e4f2e5d0a3bca as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/802c3310b71541198a6e4f2e5d0a3bca 2024-11-11T07:31:23,611 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T07:31:23,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:23,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:23,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:23,611 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:23,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:23,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/802c3310b71541198a6e4f2e5d0a3bca, entries=200, sequenceid=107, filesize=14.0 K 2024-11-11T07:31:23,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/a097971806514ce7b7af81cd7db44668 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/a097971806514ce7b7af81cd7db44668 2024-11-11T07:31:23,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/a097971806514ce7b7af81cd7db44668, entries=150, sequenceid=107, filesize=11.7 K 2024-11-11T07:31:23,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/933c77e550514821984a598e08848d77 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/933c77e550514821984a598e08848d77 2024-11-11T07:31:23,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/933c77e550514821984a598e08848d77, entries=150, sequenceid=107, filesize=11.7 K 2024-11-11T07:31:23,624 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 752742eb661c603f835a96396e7673f2 in 1046ms, sequenceid=107, compaction requested=false 2024-11-11T07:31:23,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:23,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-11T07:31:23,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:23,764 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=127.47 KB heapSize=355.69 KB 2024-11-11T07:31:23,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:23,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:23,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/705365d123da473687dd5090ccf835e0 is 50, key is test_row_0/A:col10/1731310282869/Put/seqid=0 2024-11-11T07:31:23,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742356_1532 (size=12151) 2024-11-11T07:31:24,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:24,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:24,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310344062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,080 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310344077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,098 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310344078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,099 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310344079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310344080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,104 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,183 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/705365d123da473687dd5090ccf835e0 2024-11-11T07:31:24,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/08a4d91eb78d48fca987abd890f0ba84 is 50, key is test_row_0/B:col10/1731310282869/Put/seqid=0 2024-11-11T07:31:24,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T07:31:24,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310344185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742357_1533 (size=12151) 2024-11-11T07:31:24,205 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/08a4d91eb78d48fca987abd890f0ba84 2024-11-11T07:31:24,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/c5090ac8f5904427a931339e414e6233 is 50, key is test_row_0/C:col10/1731310282869/Put/seqid=0 2024-11-11T07:31:24,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310344204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,224 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,224 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310344204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310344215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,225 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,226 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310344215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,226 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742358_1534 (size=12151) 2024-11-11T07:31:24,264 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=144 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/c5090ac8f5904427a931339e414e6233 2024-11-11T07:31:24,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/705365d123da473687dd5090ccf835e0 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/705365d123da473687dd5090ccf835e0 2024-11-11T07:31:24,274 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/705365d123da473687dd5090ccf835e0, entries=150, sequenceid=144, filesize=11.9 K 2024-11-11T07:31:24,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/08a4d91eb78d48fca987abd890f0ba84 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/08a4d91eb78d48fca987abd890f0ba84 2024-11-11T07:31:24,282 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/08a4d91eb78d48fca987abd890f0ba84, entries=150, sequenceid=144, filesize=11.9 K 2024-11-11T07:31:24,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/c5090ac8f5904427a931339e414e6233 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/c5090ac8f5904427a931339e414e6233 2024-11-11T07:31:24,287 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/c5090ac8f5904427a931339e414e6233, entries=150, sequenceid=144, filesize=11.9 K 2024-11-11T07:31:24,288 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~355.64 KB/364176, currentSize=60.38 KB/61830 for 752742eb661c603f835a96396e7673f2 in 523ms, sequenceid=144, compaction requested=true 2024-11-11T07:31:24,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:24,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:24,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-11T07:31:24,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-11T07:31:24,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-11T07:31:24,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2400 sec 2024-11-11T07:31:24,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.2460 sec 2024-11-11T07:31:24,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T07:31:24,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:24,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:24,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:24,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:24,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:24,416 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:24,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/cf4555952ea646cb9b1b200d14f9bba5 is 50, key is test_row_0/A:col10/1731310284077/Put/seqid=0 2024-11-11T07:31:24,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742359_1535 (size=14541) 2024-11-11T07:31:24,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310344495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310344497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310344497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,517 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310344497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310344499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,518 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310344614, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,629 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310344627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310344627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310344627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310344627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,845 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310344835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,845 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,853 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/cf4555952ea646cb9b1b200d14f9bba5 2024-11-11T07:31:24,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310344846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310344846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,858 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310344846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,859 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,862 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310344854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:24,863 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:24,865 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/0fae72c8f7c9455d86833c3a2a7dfc2c is 50, key is test_row_0/B:col10/1731310284077/Put/seqid=0 2024-11-11T07:31:24,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742360_1536 (size=12151) 2024-11-11T07:31:25,167 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310345154, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:25,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:25,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310345165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:25,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310345165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:25,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:25,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310345165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,178 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:25,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:25,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310345174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,187 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:25,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/0fae72c8f7c9455d86833c3a2a7dfc2c 2024-11-11T07:31:25,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/526c99f07def409e866a63b093623eac is 50, key is test_row_0/C:col10/1731310284077/Put/seqid=0 2024-11-11T07:31:25,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742361_1537 (size=12151) 2024-11-11T07:31:25,336 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/526c99f07def409e866a63b093623eac 2024-11-11T07:31:25,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/cf4555952ea646cb9b1b200d14f9bba5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/cf4555952ea646cb9b1b200d14f9bba5 2024-11-11T07:31:25,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/cf4555952ea646cb9b1b200d14f9bba5, entries=200, sequenceid=157, filesize=14.2 K 2024-11-11T07:31:25,359 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/0fae72c8f7c9455d86833c3a2a7dfc2c as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0fae72c8f7c9455d86833c3a2a7dfc2c 2024-11-11T07:31:25,364 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0fae72c8f7c9455d86833c3a2a7dfc2c, entries=150, sequenceid=157, filesize=11.9 K 2024-11-11T07:31:25,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/526c99f07def409e866a63b093623eac as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/526c99f07def409e866a63b093623eac 2024-11-11T07:31:25,371 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/526c99f07def409e866a63b093623eac, entries=150, sequenceid=157, filesize=11.9 K 2024-11-11T07:31:25,373 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 752742eb661c603f835a96396e7673f2 in 958ms, sequenceid=157, compaction requested=true 2024-11-11T07:31:25,373 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:25,373 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:25,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:25,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:25,374 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:25,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:25,374 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:25,374 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53172 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:25,374 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/A is initiating minor compaction (all files) 2024-11-11T07:31:25,374 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/A in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:25,375 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/508a6fde64754edd85203947da483c75, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/802c3310b71541198a6e4f2e5d0a3bca, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/705365d123da473687dd5090ccf835e0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/cf4555952ea646cb9b1b200d14f9bba5] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=51.9 K 2024-11-11T07:31:25,375 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 508a6fde64754edd85203947da483c75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731310281456 2024-11-11T07:31:25,375 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:25,375 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 802c3310b71541198a6e4f2e5d0a3bca, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731310281599 2024-11-11T07:31:25,375 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:25,375 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/B is initiating minor compaction (all files) 2024-11-11T07:31:25,375 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/B in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:25,376 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/fe9c1db1950f428fb237c9889253c5c7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/a097971806514ce7b7af81cd7db44668, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/08a4d91eb78d48fca987abd890f0ba84, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0fae72c8f7c9455d86833c3a2a7dfc2c] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=47.3 K 2024-11-11T07:31:25,376 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting fe9c1db1950f428fb237c9889253c5c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731310281456 2024-11-11T07:31:25,376 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 705365d123da473687dd5090ccf835e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731310282856 2024-11-11T07:31:25,376 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting a097971806514ce7b7af81cd7db44668, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731310281610 2024-11-11T07:31:25,376 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf4555952ea646cb9b1b200d14f9bba5, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731310284074 2024-11-11T07:31:25,376 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 08a4d91eb78d48fca987abd890f0ba84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731310282856 2024-11-11T07:31:25,377 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 0fae72c8f7c9455d86833c3a2a7dfc2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731310284074 2024-11-11T07:31:25,377 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:25,390 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#B#compaction#451 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:25,391 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/d259c81ec04647cd861d95aed626a3fb is 50, key is test_row_0/B:col10/1731310284077/Put/seqid=0 2024-11-11T07:31:25,400 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#A#compaction#452 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:25,401 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/09fc52699fd74ab58997da2ad717cb76 is 50, key is test_row_0/A:col10/1731310284077/Put/seqid=0 2024-11-11T07:31:25,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742362_1538 (size=12425) 2024-11-11T07:31:25,420 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/d259c81ec04647cd861d95aed626a3fb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/d259c81ec04647cd861d95aed626a3fb 2024-11-11T07:31:25,424 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 752742eb661c603f835a96396e7673f2/B of 752742eb661c603f835a96396e7673f2 into d259c81ec04647cd861d95aed626a3fb(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:25,424 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:25,424 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/B, priority=12, startTime=1731310285374; duration=0sec 2024-11-11T07:31:25,424 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:25,424 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:B 2024-11-11T07:31:25,424 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:25,425 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48442 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:25,425 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/C is initiating minor compaction (all files) 2024-11-11T07:31:25,425 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/C in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:25,425 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/aeb1f3ab761446ba9994a6a6e7cf8af3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/933c77e550514821984a598e08848d77, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/c5090ac8f5904427a931339e414e6233, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/526c99f07def409e866a63b093623eac] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=47.3 K 2024-11-11T07:31:25,426 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting aeb1f3ab761446ba9994a6a6e7cf8af3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731310281456 2024-11-11T07:31:25,426 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 933c77e550514821984a598e08848d77, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1731310281610 2024-11-11T07:31:25,426 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting c5090ac8f5904427a931339e414e6233, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=144, earliestPutTs=1731310282856 2024-11-11T07:31:25,426 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 526c99f07def409e866a63b093623eac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731310284074 2024-11-11T07:31:25,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742363_1539 (size=12425) 2024-11-11T07:31:25,438 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/09fc52699fd74ab58997da2ad717cb76 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/09fc52699fd74ab58997da2ad717cb76 2024-11-11T07:31:25,439 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#C#compaction#453 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:25,440 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/41e39d0e69e2463c907090ecfb8464a7 is 50, key is test_row_0/C:col10/1731310284077/Put/seqid=0 2024-11-11T07:31:25,451 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 752742eb661c603f835a96396e7673f2/A of 752742eb661c603f835a96396e7673f2 into 09fc52699fd74ab58997da2ad717cb76(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:25,451 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:25,451 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/A, priority=12, startTime=1731310285373; duration=0sec 2024-11-11T07:31:25,451 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:25,451 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:25,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742364_1540 (size=12425) 2024-11-11T07:31:25,470 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/41e39d0e69e2463c907090ecfb8464a7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/41e39d0e69e2463c907090ecfb8464a7 2024-11-11T07:31:25,479 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 752742eb661c603f835a96396e7673f2/C of 752742eb661c603f835a96396e7673f2 into 41e39d0e69e2463c907090ecfb8464a7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:25,479 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:25,479 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/C, priority=12, startTime=1731310285374; duration=0sec 2024-11-11T07:31:25,479 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:25,479 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:C 2024-11-11T07:31:25,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:25,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:31:25,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:25,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:25,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:25,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:25,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:25,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:25,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/7094ae39a54a40069c1412ea19f9a822 is 50, key is test_row_0/A:col10/1731310285682/Put/seqid=0 2024-11-11T07:31:25,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742365_1541 (size=14541) 2024-11-11T07:31:25,704 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/7094ae39a54a40069c1412ea19f9a822 2024-11-11T07:31:25,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:25,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310345706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:25,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,711 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:25,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/ce2fff3e5a044496bbc0fb31a705c963 is 50, key is test_row_0/B:col10/1731310285682/Put/seqid=0 2024-11-11T07:31:25,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310345706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:25,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:25,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310345708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:25,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:25,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310345708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,723 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:25,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:25,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310345710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,724 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,725 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:25,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742366_1542 (size=12151) 2024-11-11T07:31:25,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:25,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310345816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:25,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,824 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:25,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:25,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310345835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,837 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310345835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:25,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:25,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310345835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:25,838 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:25,838 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310345835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:25,839 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310346034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,049 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310346045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310346045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310346048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310346049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,061 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,062 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,151 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/ce2fff3e5a044496bbc0fb31a705c963 2024-11-11T07:31:26,177 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/8977f86c10b24ab0b7993d0ba1548e59 is 50, key is test_row_0/C:col10/1731310285682/Put/seqid=0 2024-11-11T07:31:26,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742367_1543 (size=12151) 2024-11-11T07:31:26,196 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/8977f86c10b24ab0b7993d0ba1548e59 2024-11-11T07:31:26,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/7094ae39a54a40069c1412ea19f9a822 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/7094ae39a54a40069c1412ea19f9a822 2024-11-11T07:31:26,207 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/7094ae39a54a40069c1412ea19f9a822, entries=200, sequenceid=180, filesize=14.2 K 2024-11-11T07:31:26,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/ce2fff3e5a044496bbc0fb31a705c963 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ce2fff3e5a044496bbc0fb31a705c963 2024-11-11T07:31:26,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-11T07:31:26,216 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:26,217 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:26,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ce2fff3e5a044496bbc0fb31a705c963, entries=150, sequenceid=180, filesize=11.9 K 2024-11-11T07:31:26,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-11T07:31:26,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/8977f86c10b24ab0b7993d0ba1548e59 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8977f86c10b24ab0b7993d0ba1548e59 2024-11-11T07:31:26,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T07:31:26,221 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:26,222 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:26,222 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:26,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8977f86c10b24ab0b7993d0ba1548e59, entries=150, sequenceid=180, filesize=11.9 K 2024-11-11T07:31:26,228 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 752742eb661c603f835a96396e7673f2 in 543ms, sequenceid=180, compaction requested=false 2024-11-11T07:31:26,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:26,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T07:31:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:26,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T07:31:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:26,368 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/1ebec904021646999a1a466d04806633 is 50, key is test_row_0/A:col10/1731310285706/Put/seqid=0 2024-11-11T07:31:26,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742368_1544 (size=12151) 2024-11-11T07:31:26,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T07:31:26,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:26,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:26,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:26,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:26,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:26,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:26,436 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310346424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,436 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,442 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310346430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,443 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310346433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,445 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310346436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310346436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,454 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T07:31:26,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:26,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:26,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:26,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:26,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T07:31:26,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:26,554 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310346545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,555 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,565 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310346554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310346554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,577 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310346565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310346565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,579 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T07:31:26,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:26,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:26,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:26,688 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:26,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:26,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:26,768 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310346765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,768 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,779 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/1ebec904021646999a1a466d04806633 2024-11-11T07:31:26,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310346775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,782 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310346775, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310346785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,793 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310346785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:26,799 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:26,824 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/16a94cfc094a479285e5c47f632bb347 is 50, key is test_row_0/B:col10/1731310285706/Put/seqid=0 2024-11-11T07:31:26,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T07:31:26,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:26,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:26,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:26,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:26,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:26,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:26,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T07:31:26,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742369_1545 (size=12151) 2024-11-11T07:31:26,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/16a94cfc094a479285e5c47f632bb347 2024-11-11T07:31:26,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/70ade268a79a44dab46512cdbb3ccc56 is 50, key is test_row_0/C:col10/1731310285706/Put/seqid=0 2024-11-11T07:31:26,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742370_1546 (size=12151) 2024-11-11T07:31:26,892 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/70ade268a79a44dab46512cdbb3ccc56 2024-11-11T07:31:26,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/1ebec904021646999a1a466d04806633 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1ebec904021646999a1a466d04806633 2024-11-11T07:31:26,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1ebec904021646999a1a466d04806633, entries=150, sequenceid=193, filesize=11.9 K 2024-11-11T07:31:26,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/16a94cfc094a479285e5c47f632bb347 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/16a94cfc094a479285e5c47f632bb347 2024-11-11T07:31:26,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/16a94cfc094a479285e5c47f632bb347, entries=150, sequenceid=193, filesize=11.9 K 2024-11-11T07:31:26,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/70ade268a79a44dab46512cdbb3ccc56 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/70ade268a79a44dab46512cdbb3ccc56 2024-11-11T07:31:26,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/70ade268a79a44dab46512cdbb3ccc56, entries=150, sequenceid=193, filesize=11.9 K 2024-11-11T07:31:26,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 752742eb661c603f835a96396e7673f2 in 559ms, sequenceid=193, compaction requested=true 2024-11-11T07:31:26,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:26,919 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:26,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:26,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:26,919 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:26,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:26,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:26,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:26,920 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:26,920 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39117 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:26,921 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/A is initiating minor compaction (all files) 2024-11-11T07:31:26,921 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/A in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:26,921 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/09fc52699fd74ab58997da2ad717cb76, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/7094ae39a54a40069c1412ea19f9a822, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1ebec904021646999a1a466d04806633] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=38.2 K 2024-11-11T07:31:26,921 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:26,921 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/B is initiating minor compaction (all files) 2024-11-11T07:31:26,921 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/B in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:26,921 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/d259c81ec04647cd861d95aed626a3fb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ce2fff3e5a044496bbc0fb31a705c963, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/16a94cfc094a479285e5c47f632bb347] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=35.9 K 2024-11-11T07:31:26,921 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting d259c81ec04647cd861d95aed626a3fb, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731310284074 2024-11-11T07:31:26,922 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09fc52699fd74ab58997da2ad717cb76, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731310284074 2024-11-11T07:31:26,922 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting ce2fff3e5a044496bbc0fb31a705c963, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731310284494 2024-11-11T07:31:26,922 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7094ae39a54a40069c1412ea19f9a822, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731310284493 2024-11-11T07:31:26,922 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 16a94cfc094a479285e5c47f632bb347, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731310285706 2024-11-11T07:31:26,922 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ebec904021646999a1a466d04806633, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731310285706 2024-11-11T07:31:26,934 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#A#compaction#460 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:26,934 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/b73446016fd44eb8913a0802f081fd76 is 50, key is test_row_0/A:col10/1731310285706/Put/seqid=0 2024-11-11T07:31:26,941 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#B#compaction#461 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:26,941 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/e869c048d5064130bbd53492b15f2e19 is 50, key is test_row_0/B:col10/1731310285706/Put/seqid=0 2024-11-11T07:31:26,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742372_1548 (size=12527) 2024-11-11T07:31:26,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742371_1547 (size=12527) 2024-11-11T07:31:26,960 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/b73446016fd44eb8913a0802f081fd76 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/b73446016fd44eb8913a0802f081fd76 2024-11-11T07:31:26,964 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/A of 752742eb661c603f835a96396e7673f2 into b73446016fd44eb8913a0802f081fd76(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:26,964 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:26,964 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/A, priority=13, startTime=1731310286919; duration=0sec 2024-11-11T07:31:26,965 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:26,965 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:26,965 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:26,967 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36727 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:26,967 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/C is initiating minor compaction (all files) 2024-11-11T07:31:26,967 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/C in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:26,967 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/41e39d0e69e2463c907090ecfb8464a7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8977f86c10b24ab0b7993d0ba1548e59, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/70ade268a79a44dab46512cdbb3ccc56] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=35.9 K 2024-11-11T07:31:26,969 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 41e39d0e69e2463c907090ecfb8464a7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1731310284074 2024-11-11T07:31:26,970 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8977f86c10b24ab0b7993d0ba1548e59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1731310284494 2024-11-11T07:31:26,973 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70ade268a79a44dab46512cdbb3ccc56, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731310285706 2024-11-11T07:31:26,989 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#C#compaction#462 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:26,989 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/5ed89a432ea34e8098920b481b7be052 is 50, key is test_row_0/C:col10/1731310285706/Put/seqid=0 2024-11-11T07:31:26,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-11T07:31:26,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:26,997 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:31:26,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:26,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:26,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:26,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:26,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:26,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:27,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/cdca3cb0362f4328ba0773d17067a391 is 50, key is test_row_0/A:col10/1731310286423/Put/seqid=0 2024-11-11T07:31:27,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742373_1549 (size=12527) 2024-11-11T07:31:27,053 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/5ed89a432ea34e8098920b481b7be052 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/5ed89a432ea34e8098920b481b7be052 2024-11-11T07:31:27,057 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/C of 752742eb661c603f835a96396e7673f2 into 5ed89a432ea34e8098920b481b7be052(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:27,057 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:27,057 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/C, priority=13, startTime=1731310286920; duration=0sec 2024-11-11T07:31:27,057 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:27,057 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:C 2024-11-11T07:31:27,083 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742374_1550 (size=12151) 2024-11-11T07:31:27,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:27,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:27,170 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310347152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310347152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,171 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,171 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310347153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310347153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310347171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,188 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310347275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310347275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,286 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310347276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,287 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310347276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310347294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,305 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T07:31:27,360 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/e869c048d5064130bbd53492b15f2e19 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/e869c048d5064130bbd53492b15f2e19 2024-11-11T07:31:27,365 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/B of 752742eb661c603f835a96396e7673f2 into e869c048d5064130bbd53492b15f2e19(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:27,365 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:27,365 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/B, priority=13, startTime=1731310286919; duration=0sec 2024-11-11T07:31:27,365 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:27,365 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:B 2024-11-11T07:31:27,484 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/cdca3cb0362f4328ba0773d17067a391 2024-11-11T07:31:27,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/e45c3b872384449f8d1063ce809c4f7e is 50, key is test_row_0/B:col10/1731310286423/Put/seqid=0 2024-11-11T07:31:27,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310347496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,505 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310347496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,506 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310347496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,506 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310347496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742375_1551 (size=12151) 2024-11-11T07:31:27,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310347516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,522 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,523 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/e45c3b872384449f8d1063ce809c4f7e 2024-11-11T07:31:27,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/ba1b173af7d94d6ebf1f395f939a4161 is 50, key is test_row_0/C:col10/1731310286423/Put/seqid=0 2024-11-11T07:31:27,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742376_1552 (size=12151) 2024-11-11T07:31:27,548 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/ba1b173af7d94d6ebf1f395f939a4161 2024-11-11T07:31:27,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/cdca3cb0362f4328ba0773d17067a391 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/cdca3cb0362f4328ba0773d17067a391 2024-11-11T07:31:27,556 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/cdca3cb0362f4328ba0773d17067a391, entries=150, sequenceid=214, filesize=11.9 K 2024-11-11T07:31:27,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/e45c3b872384449f8d1063ce809c4f7e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/e45c3b872384449f8d1063ce809c4f7e 2024-11-11T07:31:27,560 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/e45c3b872384449f8d1063ce809c4f7e, entries=150, sequenceid=214, filesize=11.9 K 2024-11-11T07:31:27,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/ba1b173af7d94d6ebf1f395f939a4161 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ba1b173af7d94d6ebf1f395f939a4161 2024-11-11T07:31:27,570 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ba1b173af7d94d6ebf1f395f939a4161, entries=150, sequenceid=214, filesize=11.9 K 2024-11-11T07:31:27,571 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 752742eb661c603f835a96396e7673f2 in 574ms, sequenceid=214, compaction requested=false 2024-11-11T07:31:27,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:27,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:27,571 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-11T07:31:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-11T07:31:27,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-11T07:31:27,577 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3530 sec 2024-11-11T07:31:27,579 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.3610 sec 2024-11-11T07:31:27,822 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T07:31:27,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:27,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:27,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:27,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:27,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:27,822 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:27,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:27,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/5cb3079429a34deca3317f13f5113930 is 50, key is test_row_0/A:col10/1731310287166/Put/seqid=0 2024-11-11T07:31:27,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742377_1553 (size=14541) 2024-11-11T07:31:27,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/5cb3079429a34deca3317f13f5113930 2024-11-11T07:31:27,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/515ba2b710d8492592e1a600c325d47f is 50, key is test_row_0/B:col10/1731310287166/Put/seqid=0 2024-11-11T07:31:27,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310347861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,879 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310347864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742378_1554 (size=12151) 2024-11-11T07:31:27,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/515ba2b710d8492592e1a600c325d47f 2024-11-11T07:31:27,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310347871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,887 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310347873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,888 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310347881, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:27,889 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,890 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:27,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:27,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/faf1bdc5c1a54e5e8837745dcabfebeb is 50, key is test_row_0/C:col10/1731310287166/Put/seqid=0 2024-11-11T07:31:27,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742379_1555 (size=12151) 2024-11-11T07:31:27,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=229 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/faf1bdc5c1a54e5e8837745dcabfebeb 2024-11-11T07:31:27,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/5cb3079429a34deca3317f13f5113930 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/5cb3079429a34deca3317f13f5113930 2024-11-11T07:31:27,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/5cb3079429a34deca3317f13f5113930, entries=200, sequenceid=229, filesize=14.2 K 2024-11-11T07:31:27,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/515ba2b710d8492592e1a600c325d47f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/515ba2b710d8492592e1a600c325d47f 2024-11-11T07:31:27,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/515ba2b710d8492592e1a600c325d47f, entries=150, sequenceid=229, filesize=11.9 K 2024-11-11T07:31:27,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/faf1bdc5c1a54e5e8837745dcabfebeb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/faf1bdc5c1a54e5e8837745dcabfebeb 2024-11-11T07:31:27,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/faf1bdc5c1a54e5e8837745dcabfebeb, entries=150, sequenceid=229, filesize=11.9 K 2024-11-11T07:31:27,940 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 752742eb661c603f835a96396e7673f2 in 118ms, sequenceid=229, compaction requested=true 2024-11-11T07:31:27,940 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:27,940 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:27,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:27,940 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:27,940 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:27,941 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39219 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:27,941 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/A is initiating minor compaction (all files) 2024-11-11T07:31:27,941 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/A in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:27,941 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/b73446016fd44eb8913a0802f081fd76, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/cdca3cb0362f4328ba0773d17067a391, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/5cb3079429a34deca3317f13f5113930] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=38.3 K 2024-11-11T07:31:27,942 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting b73446016fd44eb8913a0802f081fd76, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731310285706 2024-11-11T07:31:27,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:27,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:27,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:27,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:27,944 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdca3cb0362f4328ba0773d17067a391, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731310286404 2024-11-11T07:31:27,944 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:27,945 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/B is initiating minor compaction (all files) 2024-11-11T07:31:27,945 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/B in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:27,945 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/e869c048d5064130bbd53492b15f2e19, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/e45c3b872384449f8d1063ce809c4f7e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/515ba2b710d8492592e1a600c325d47f] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=36.0 K 2024-11-11T07:31:27,945 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cb3079429a34deca3317f13f5113930, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731310287151 2024-11-11T07:31:27,945 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting e869c048d5064130bbd53492b15f2e19, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731310285706 2024-11-11T07:31:27,946 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting e45c3b872384449f8d1063ce809c4f7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731310286404 2024-11-11T07:31:27,946 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 515ba2b710d8492592e1a600c325d47f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731310287151 2024-11-11T07:31:27,966 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#A#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:27,967 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/dacaa6d953484675ab7797f2792555a0 is 50, key is test_row_0/A:col10/1731310287166/Put/seqid=0 2024-11-11T07:31:27,970 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#B#compaction#470 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:27,971 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/0254e6afb50d481ea4909a9840a655b4 is 50, key is test_row_0/B:col10/1731310287166/Put/seqid=0 2024-11-11T07:31:27,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:31:27,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:27,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:27,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:27,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:27,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:27,996 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:27,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:28,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742380_1556 (size=12629) 2024-11-11T07:31:28,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/d8f3347ddb584684a50d11c2dc4b593d is 50, key is test_row_0/A:col10/1731310287870/Put/seqid=0 2024-11-11T07:31:28,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742381_1557 (size=12629) 2024-11-11T07:31:28,023 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/0254e6afb50d481ea4909a9840a655b4 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0254e6afb50d481ea4909a9840a655b4 2024-11-11T07:31:28,028 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/B of 752742eb661c603f835a96396e7673f2 into 0254e6afb50d481ea4909a9840a655b4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:28,028 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:28,028 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/B, priority=13, startTime=1731310287940; duration=0sec 2024-11-11T07:31:28,028 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:28,028 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:B 2024-11-11T07:31:28,028 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:28,029 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:28,029 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/C is initiating minor compaction (all files) 2024-11-11T07:31:28,029 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/C in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:28,030 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/5ed89a432ea34e8098920b481b7be052, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ba1b173af7d94d6ebf1f395f939a4161, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/faf1bdc5c1a54e5e8837745dcabfebeb] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=36.0 K 2024-11-11T07:31:28,030 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ed89a432ea34e8098920b481b7be052, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731310285706 2024-11-11T07:31:28,030 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting ba1b173af7d94d6ebf1f395f939a4161, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1731310286404 2024-11-11T07:31:28,030 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting faf1bdc5c1a54e5e8837745dcabfebeb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731310287151 2024-11-11T07:31:28,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310348017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310348017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,041 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310348036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,042 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#C#compaction#472 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:28,043 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/587dc6c96cb6401999fe17dfc0fc5e7c is 50, key is test_row_0/C:col10/1731310287166/Put/seqid=0 2024-11-11T07:31:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,044 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742383_1559 (size=12629) 2024-11-11T07:31:28,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742382_1558 (size=14541) 2024-11-11T07:31:28,051 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/587dc6c96cb6401999fe17dfc0fc5e7c as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/587dc6c96cb6401999fe17dfc0fc5e7c 2024-11-11T07:31:28,052 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/d8f3347ddb584684a50d11c2dc4b593d 2024-11-11T07:31:28,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,062 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/C of 752742eb661c603f835a96396e7673f2 into 587dc6c96cb6401999fe17dfc0fc5e7c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:28,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310348037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,062 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:28,062 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/C, priority=13, startTime=1731310287942; duration=0sec 2024-11-11T07:31:28,062 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:28,062 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:C 2024-11-11T07:31:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310348043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,063 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/72255fd6f6b04e5e904ce55655f0d539 is 50, key is test_row_0/B:col10/1731310287870/Put/seqid=0 2024-11-11T07:31:28,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742384_1560 (size=12151) 2024-11-11T07:31:28,107 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/72255fd6f6b04e5e904ce55655f0d539 2024-11-11T07:31:28,127 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/ef4a679fdb534b1093386ecda30485ac is 50, key is test_row_0/C:col10/1731310287870/Put/seqid=0 2024-11-11T07:31:28,151 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310348144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,152 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310348155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,162 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310348155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310348176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310348176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,180 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742385_1561 (size=12151) 2024-11-11T07:31:28,187 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=249 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/ef4a679fdb534b1093386ecda30485ac 2024-11-11T07:31:28,193 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/d8f3347ddb584684a50d11c2dc4b593d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/d8f3347ddb584684a50d11c2dc4b593d 2024-11-11T07:31:28,196 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/d8f3347ddb584684a50d11c2dc4b593d, entries=200, sequenceid=249, filesize=14.2 K 2024-11-11T07:31:28,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/72255fd6f6b04e5e904ce55655f0d539 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/72255fd6f6b04e5e904ce55655f0d539 2024-11-11T07:31:28,200 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/72255fd6f6b04e5e904ce55655f0d539, entries=150, sequenceid=249, filesize=11.9 K 2024-11-11T07:31:28,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/ef4a679fdb534b1093386ecda30485ac as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ef4a679fdb534b1093386ecda30485ac 2024-11-11T07:31:28,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ef4a679fdb534b1093386ecda30485ac, entries=150, sequenceid=249, filesize=11.9 K 2024-11-11T07:31:28,213 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-11T07:31:28,217 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 752742eb661c603f835a96396e7673f2 in 222ms, sequenceid=249, compaction requested=false 2024-11-11T07:31:28,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:28,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:28,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T07:31:28,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:28,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:28,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:28,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:28,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:28,362 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:28,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-11T07:31:28,364 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:28,365 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:28,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-11T07:31:28,367 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:28,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T07:31:28,369 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:28,369 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:28,377 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/1503bad9045f4b5db163acfd1daf080a is 50, key is test_row_0/A:col10/1731310288036/Put/seqid=0 2024-11-11T07:31:28,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742386_1562 (size=14741) 2024-11-11T07:31:28,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/1503bad9045f4b5db163acfd1daf080a 2024-11-11T07:31:28,408 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/386c48fc723e48b0b189af6d4244235a is 50, key is test_row_0/B:col10/1731310288036/Put/seqid=0 2024-11-11T07:31:28,414 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/dacaa6d953484675ab7797f2792555a0 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/dacaa6d953484675ab7797f2792555a0 2024-11-11T07:31:28,427 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/A of 752742eb661c603f835a96396e7673f2 into dacaa6d953484675ab7797f2792555a0(size=12.3 K), total size for store is 26.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:28,427 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:28,427 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/A, priority=13, startTime=1731310287940; duration=0sec 2024-11-11T07:31:28,428 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:28,428 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:28,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310348420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,450 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310348420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,451 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,452 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310348438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,462 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742387_1563 (size=12301) 2024-11-11T07:31:28,471 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/386c48fc723e48b0b189af6d4244235a 2024-11-11T07:31:28,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T07:31:28,478 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310348449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310348450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,480 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,487 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/599b985472af40b68b4def1ada1d5c30 is 50, key is test_row_0/C:col10/1731310288036/Put/seqid=0 2024-11-11T07:31:28,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742388_1564 (size=12301) 2024-11-11T07:31:28,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T07:31:28,522 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=264 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/599b985472af40b68b4def1ada1d5c30 2024-11-11T07:31:28,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:28,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:28,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:28,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:28,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:28,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:28,542 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/1503bad9045f4b5db163acfd1daf080a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1503bad9045f4b5db163acfd1daf080a 2024-11-11T07:31:28,552 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1503bad9045f4b5db163acfd1daf080a, entries=200, sequenceid=264, filesize=14.4 K 2024-11-11T07:31:28,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/386c48fc723e48b0b189af6d4244235a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/386c48fc723e48b0b189af6d4244235a 2024-11-11T07:31:28,557 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/386c48fc723e48b0b189af6d4244235a, entries=150, sequenceid=264, filesize=12.0 K 2024-11-11T07:31:28,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/599b985472af40b68b4def1ada1d5c30 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/599b985472af40b68b4def1ada1d5c30 2024-11-11T07:31:28,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/599b985472af40b68b4def1ada1d5c30, entries=150, sequenceid=264, filesize=12.0 K 2024-11-11T07:31:28,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 752742eb661c603f835a96396e7673f2 in 207ms, sequenceid=264, compaction requested=true 2024-11-11T07:31:28,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:28,570 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:28,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:28,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:28,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:28,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:28,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:28,571 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:31:28,572 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:28,572 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41911 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:28,572 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/A is initiating minor compaction (all files) 2024-11-11T07:31:28,572 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/A in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:28,573 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/dacaa6d953484675ab7797f2792555a0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/d8f3347ddb584684a50d11c2dc4b593d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1503bad9045f4b5db163acfd1daf080a] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=40.9 K 2024-11-11T07:31:28,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:28,573 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:28,574 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/B is initiating minor compaction (all files) 2024-11-11T07:31:28,574 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/B in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:28,574 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0254e6afb50d481ea4909a9840a655b4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/72255fd6f6b04e5e904ce55655f0d539, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/386c48fc723e48b0b189af6d4244235a] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=36.2 K 2024-11-11T07:31:28,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-11T07:31:28,574 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting dacaa6d953484675ab7797f2792555a0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731310287151 2024-11-11T07:31:28,574 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0254e6afb50d481ea4909a9840a655b4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731310287151 2024-11-11T07:31:28,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:28,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:28,574 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting d8f3347ddb584684a50d11c2dc4b593d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731310287859 2024-11-11T07:31:28,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:28,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:28,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:28,574 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:28,575 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72255fd6f6b04e5e904ce55655f0d539, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731310287859 2024-11-11T07:31:28,575 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 1503bad9045f4b5db163acfd1daf080a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731310288034 2024-11-11T07:31:28,575 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 386c48fc723e48b0b189af6d4244235a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731310288034 2024-11-11T07:31:28,585 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#A#compaction#478 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:28,585 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/5163ed8867c347e99d1384e84e3ffd73 is 50, key is test_row_0/A:col10/1731310288036/Put/seqid=0 2024-11-11T07:31:28,593 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/a2017fa3eac747fd916955459d871bba is 50, key is test_row_0/A:col10/1731310288445/Put/seqid=0 2024-11-11T07:31:28,597 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#B#compaction#480 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:28,598 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/100de80240014a439bf2121778a70550 is 50, key is test_row_0/B:col10/1731310288036/Put/seqid=0 2024-11-11T07:31:28,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742390_1566 (size=17181) 2024-11-11T07:31:28,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742391_1567 (size=12881) 2024-11-11T07:31:28,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742389_1565 (size=12881) 2024-11-11T07:31:28,650 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310348632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310348632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,651 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310348632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310348634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,652 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310348634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,654 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T07:31:28,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:28,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:28,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:28,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:28,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:28,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:28,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T07:31:28,763 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310348755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,764 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310348755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,764 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,765 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310348755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,765 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310348764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310348764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,832 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T07:31:28,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:28,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:28,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:28,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:28,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:28,985 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310348975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,986 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310348975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,986 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310348975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310348975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310348976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,991 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:28,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:28,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T07:31:28,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:28,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:28,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:28,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:28,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:28,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T07:31:29,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/a2017fa3eac747fd916955459d871bba 2024-11-11T07:31:29,018 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/100de80240014a439bf2121778a70550 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/100de80240014a439bf2121778a70550 2024-11-11T07:31:29,018 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/5163ed8867c347e99d1384e84e3ffd73 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/5163ed8867c347e99d1384e84e3ffd73 2024-11-11T07:31:29,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/f64752d3383d49278251bd2825da54cb is 50, key is test_row_0/B:col10/1731310288445/Put/seqid=0 2024-11-11T07:31:29,025 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/A of 752742eb661c603f835a96396e7673f2 into 5163ed8867c347e99d1384e84e3ffd73(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:29,025 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:29,025 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/B of 752742eb661c603f835a96396e7673f2 into 100de80240014a439bf2121778a70550(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:29,025 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:29,025 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/A, priority=13, startTime=1731310288570; duration=0sec 2024-11-11T07:31:29,026 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/B, priority=13, startTime=1731310288571; duration=0sec 2024-11-11T07:31:29,026 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:29,026 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:29,026 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:29,026 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:B 2024-11-11T07:31:29,026 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:29,026 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37081 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:29,027 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/C is initiating minor compaction (all files) 2024-11-11T07:31:29,027 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/C in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:29,027 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/587dc6c96cb6401999fe17dfc0fc5e7c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ef4a679fdb534b1093386ecda30485ac, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/599b985472af40b68b4def1ada1d5c30] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=36.2 K 2024-11-11T07:31:29,027 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 587dc6c96cb6401999fe17dfc0fc5e7c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=229, earliestPutTs=1731310287151 2024-11-11T07:31:29,027 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting ef4a679fdb534b1093386ecda30485ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=249, earliestPutTs=1731310287859 2024-11-11T07:31:29,027 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 599b985472af40b68b4def1ada1d5c30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731310288034 2024-11-11T07:31:29,052 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#C#compaction#482 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:29,053 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/06e5f3bc4e6a49518551f9d31e926301 is 50, key is test_row_0/C:col10/1731310288036/Put/seqid=0 2024-11-11T07:31:29,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742392_1568 (size=12301) 2024-11-11T07:31:29,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742393_1569 (size=12881) 2024-11-11T07:31:29,115 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/06e5f3bc4e6a49518551f9d31e926301 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/06e5f3bc4e6a49518551f9d31e926301 2024-11-11T07:31:29,131 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/C of 752742eb661c603f835a96396e7673f2 into 06e5f3bc4e6a49518551f9d31e926301(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:29,131 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:29,131 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/C, priority=13, startTime=1731310288571; duration=0sec 2024-11-11T07:31:29,131 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:29,131 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:C 2024-11-11T07:31:29,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T07:31:29,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:29,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:29,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:29,146 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:29,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:29,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:29,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T07:31:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:29,299 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:29,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:29,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:29,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310349295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:29,303 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310349295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:29,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310349295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,304 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:29,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:29,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310349295, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:29,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,305 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:29,311 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:29,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310349305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,311 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:29,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T07:31:29,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:29,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:29,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:29,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:29,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:29,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:29,497 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/f64752d3383d49278251bd2825da54cb 2024-11-11T07:31:29,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T07:31:29,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/77eda05fb40f4f23b7a4394fadc45400 is 50, key is test_row_0/C:col10/1731310288445/Put/seqid=0 2024-11-11T07:31:29,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742394_1570 (size=12301) 2024-11-11T07:31:29,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/77eda05fb40f4f23b7a4394fadc45400 2024-11-11T07:31:29,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/a2017fa3eac747fd916955459d871bba as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/a2017fa3eac747fd916955459d871bba 2024-11-11T07:31:29,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/a2017fa3eac747fd916955459d871bba, entries=250, sequenceid=286, filesize=16.8 K 2024-11-11T07:31:29,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/f64752d3383d49278251bd2825da54cb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f64752d3383d49278251bd2825da54cb 2024-11-11T07:31:29,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f64752d3383d49278251bd2825da54cb, entries=150, sequenceid=286, filesize=12.0 K 2024-11-11T07:31:29,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/77eda05fb40f4f23b7a4394fadc45400 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/77eda05fb40f4f23b7a4394fadc45400 2024-11-11T07:31:29,584 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/77eda05fb40f4f23b7a4394fadc45400, entries=150, sequenceid=286, filesize=12.0 K 2024-11-11T07:31:29,585 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 752742eb661c603f835a96396e7673f2 in 1011ms, sequenceid=286, compaction requested=false 2024-11-11T07:31:29,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:29,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-11T07:31:29,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:29,606 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T07:31:29,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:29,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:29,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:29,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:29,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:29,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:29,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/feb2df4ceff9471ea7146d38d918b035 is 50, key is test_row_0/A:col10/1731310288631/Put/seqid=0 2024-11-11T07:31:29,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742395_1571 (size=12301) 2024-11-11T07:31:29,660 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/feb2df4ceff9471ea7146d38d918b035 2024-11-11T07:31:29,673 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/c4389f0cbd354c3d8a7439e8b2658455 is 50, key is test_row_0/B:col10/1731310288631/Put/seqid=0 2024-11-11T07:31:29,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742396_1572 (size=12301) 2024-11-11T07:31:29,712 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/c4389f0cbd354c3d8a7439e8b2658455 2024-11-11T07:31:29,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/de419c5566ea4085b15fdb74cecd6946 is 50, key is test_row_0/C:col10/1731310288631/Put/seqid=0 2024-11-11T07:31:29,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742397_1573 (size=12301) 2024-11-11T07:31:29,735 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/de419c5566ea4085b15fdb74cecd6946 2024-11-11T07:31:29,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/feb2df4ceff9471ea7146d38d918b035 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/feb2df4ceff9471ea7146d38d918b035 2024-11-11T07:31:29,758 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/feb2df4ceff9471ea7146d38d918b035, entries=150, sequenceid=300, filesize=12.0 K 2024-11-11T07:31:29,758 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/c4389f0cbd354c3d8a7439e8b2658455 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/c4389f0cbd354c3d8a7439e8b2658455 2024-11-11T07:31:29,767 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/c4389f0cbd354c3d8a7439e8b2658455, entries=150, sequenceid=300, filesize=12.0 K 2024-11-11T07:31:29,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/de419c5566ea4085b15fdb74cecd6946 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/de419c5566ea4085b15fdb74cecd6946 2024-11-11T07:31:29,775 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/de419c5566ea4085b15fdb74cecd6946, entries=150, sequenceid=300, filesize=12.0 K 2024-11-11T07:31:29,775 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=0 B/0 for 752742eb661c603f835a96396e7673f2 in 169ms, sequenceid=300, compaction requested=true 2024-11-11T07:31:29,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:29,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:29,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-11T07:31:29,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-11T07:31:29,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-11T07:31:29,795 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4170 sec 2024-11-11T07:31:29,797 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.4310 sec 2024-11-11T07:31:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:29,876 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T07:31:29,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:29,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:29,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:29,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:29,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:29,877 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:29,884 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/8e94d36673c04896a4f7acbd8f4be344 is 50, key is test_row_0/A:col10/1731310289838/Put/seqid=0 2024-11-11T07:31:29,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742398_1574 (size=12301) 2024-11-11T07:31:29,955 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:29,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310349942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,956 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:29,957 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310349943, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:29,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310349944, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:29,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310349957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,961 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:29,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310349957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:29,961 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,069 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310350066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,069 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310350066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,070 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310350066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,070 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310350066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,071 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310350066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,071 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310350275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310350275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310350276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310350276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310350276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,302 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/8e94d36673c04896a4f7acbd8f4be344 2024-11-11T07:31:30,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/450a5d14f617463cac8a815b121c0198 is 50, key is test_row_0/B:col10/1731310289838/Put/seqid=0 2024-11-11T07:31:30,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742399_1575 (size=12301) 2024-11-11T07:31:30,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/450a5d14f617463cac8a815b121c0198 2024-11-11T07:31:30,361 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/8c0bbd74b7a647e1b5a6b67d03390cf4 is 50, key is test_row_0/C:col10/1731310289838/Put/seqid=0 2024-11-11T07:31:30,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742400_1576 (size=12301) 2024-11-11T07:31:30,380 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/8c0bbd74b7a647e1b5a6b67d03390cf4 2024-11-11T07:31:30,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/8e94d36673c04896a4f7acbd8f4be344 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/8e94d36673c04896a4f7acbd8f4be344 2024-11-11T07:31:30,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/8e94d36673c04896a4f7acbd8f4be344, entries=150, sequenceid=310, filesize=12.0 K 2024-11-11T07:31:30,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/450a5d14f617463cac8a815b121c0198 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/450a5d14f617463cac8a815b121c0198 2024-11-11T07:31:30,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/450a5d14f617463cac8a815b121c0198, entries=150, sequenceid=310, filesize=12.0 K 2024-11-11T07:31:30,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/8c0bbd74b7a647e1b5a6b67d03390cf4 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8c0bbd74b7a647e1b5a6b67d03390cf4 2024-11-11T07:31:30,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8c0bbd74b7a647e1b5a6b67d03390cf4, entries=150, sequenceid=310, filesize=12.0 K 2024-11-11T07:31:30,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=127.47 KB/130530 for 752742eb661c603f835a96396e7673f2 in 530ms, sequenceid=310, compaction requested=true 2024-11-11T07:31:30,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:30,406 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:30,407 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54664 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:30,407 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/A is initiating minor compaction (all files) 2024-11-11T07:31:30,407 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/A in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:30,407 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/5163ed8867c347e99d1384e84e3ffd73, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/a2017fa3eac747fd916955459d871bba, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/feb2df4ceff9471ea7146d38d918b035, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/8e94d36673c04896a4f7acbd8f4be344] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=53.4 K 2024-11-11T07:31:30,408 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5163ed8867c347e99d1384e84e3ffd73, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731310288034 2024-11-11T07:31:30,408 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2017fa3eac747fd916955459d871bba, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731310288395 2024-11-11T07:31:30,408 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting feb2df4ceff9471ea7146d38d918b035, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731310288589 2024-11-11T07:31:30,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:30,408 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8e94d36673c04896a4f7acbd8f4be344, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731310289820 2024-11-11T07:31:30,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:30,410 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:30,410 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:30,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:30,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:30,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:30,411 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:30,411 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/B is initiating minor compaction (all files) 2024-11-11T07:31:30,411 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/B in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:30,411 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/100de80240014a439bf2121778a70550, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f64752d3383d49278251bd2825da54cb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/c4389f0cbd354c3d8a7439e8b2658455, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/450a5d14f617463cac8a815b121c0198] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=48.6 K 2024-11-11T07:31:30,412 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 100de80240014a439bf2121778a70550, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731310288034 2024-11-11T07:31:30,412 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f64752d3383d49278251bd2825da54cb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731310288395 2024-11-11T07:31:30,413 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting c4389f0cbd354c3d8a7439e8b2658455, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731310288589 2024-11-11T07:31:30,413 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 450a5d14f617463cac8a815b121c0198, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731310289820 2024-11-11T07:31:30,417 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#A#compaction#490 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:30,417 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/d8d7f7625858481b9dbc61c535307092 is 50, key is test_row_0/A:col10/1731310289838/Put/seqid=0 2024-11-11T07:31:30,422 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#B#compaction#491 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:30,422 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/ac0e0a0f286f43c3980489b3a8adab3e is 50, key is test_row_0/B:col10/1731310289838/Put/seqid=0 2024-11-11T07:31:30,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742401_1577 (size=13017) 2024-11-11T07:31:30,450 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/d8d7f7625858481b9dbc61c535307092 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/d8d7f7625858481b9dbc61c535307092 2024-11-11T07:31:30,454 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 752742eb661c603f835a96396e7673f2/A of 752742eb661c603f835a96396e7673f2 into d8d7f7625858481b9dbc61c535307092(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:30,454 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:30,454 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/A, priority=12, startTime=1731310290406; duration=0sec 2024-11-11T07:31:30,455 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:30,455 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:30,455 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:30,456 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:30,456 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/C is initiating minor compaction (all files) 2024-11-11T07:31:30,456 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/C in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:30,456 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/06e5f3bc4e6a49518551f9d31e926301, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/77eda05fb40f4f23b7a4394fadc45400, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/de419c5566ea4085b15fdb74cecd6946, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8c0bbd74b7a647e1b5a6b67d03390cf4] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=48.6 K 2024-11-11T07:31:30,456 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06e5f3bc4e6a49518551f9d31e926301, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=264, earliestPutTs=1731310288034 2024-11-11T07:31:30,457 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77eda05fb40f4f23b7a4394fadc45400, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731310288395 2024-11-11T07:31:30,457 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting de419c5566ea4085b15fdb74cecd6946, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=300, earliestPutTs=1731310288589 2024-11-11T07:31:30,457 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8c0bbd74b7a647e1b5a6b67d03390cf4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731310289820 2024-11-11T07:31:30,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742402_1578 (size=13017) 2024-11-11T07:31:30,485 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#C#compaction#492 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:30,485 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/6615c9d8947f48b7ae8d1fe36c248158 is 50, key is test_row_0/C:col10/1731310289838/Put/seqid=0 2024-11-11T07:31:30,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-11T07:31:30,515 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:30,516 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-11T07:31:30,518 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:30,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T07:31:30,519 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:30,519 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:30,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742403_1579 (size=13017) 2024-11-11T07:31:30,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=134.18 KB heapSize=399.89 KB 2024-11-11T07:31:30,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:30,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:30,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:30,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:30,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:30,609 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:30,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:30,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/bf4cda33e673490dac8fd4df1fc724b6 is 50, key is test_row_0/A:col10/1731310290607/Put/seqid=0 2024-11-11T07:31:30,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T07:31:30,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310350616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310350616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310350619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310350630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,650 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310350637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742404_1580 (size=14741) 2024-11-11T07:31:30,664 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/bf4cda33e673490dac8fd4df1fc724b6 2024-11-11T07:31:30,673 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/924b8a2fb34e46069272a526909d7eca is 50, key is test_row_0/B:col10/1731310290607/Put/seqid=0 2024-11-11T07:31:30,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T07:31:30,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:30,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:30,674 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:30,675 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:30,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:30,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:30,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742405_1581 (size=12301) 2024-11-11T07:31:30,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/924b8a2fb34e46069272a526909d7eca 2024-11-11T07:31:30,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/a9ce33ca96ba43fdb9dee59d214b8b97 is 50, key is test_row_0/C:col10/1731310290607/Put/seqid=0 2024-11-11T07:31:30,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742406_1582 (size=12301) 2024-11-11T07:31:30,725 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/a9ce33ca96ba43fdb9dee59d214b8b97 2024-11-11T07:31:30,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/bf4cda33e673490dac8fd4df1fc724b6 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/bf4cda33e673490dac8fd4df1fc724b6 2024-11-11T07:31:30,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/bf4cda33e673490dac8fd4df1fc724b6, entries=200, sequenceid=334, filesize=14.4 K 2024-11-11T07:31:30,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/924b8a2fb34e46069272a526909d7eca as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/924b8a2fb34e46069272a526909d7eca 2024-11-11T07:31:30,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/924b8a2fb34e46069272a526909d7eca, entries=150, sequenceid=334, filesize=12.0 K 2024-11-11T07:31:30,741 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/a9ce33ca96ba43fdb9dee59d214b8b97 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a9ce33ca96ba43fdb9dee59d214b8b97 2024-11-11T07:31:30,748 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a9ce33ca96ba43fdb9dee59d214b8b97, entries=150, sequenceid=334, filesize=12.0 K 2024-11-11T07:31:30,749 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~399.84 KB/409440, currentSize=40.25 KB/41220 for 752742eb661c603f835a96396e7673f2 in 141ms, sequenceid=334, compaction requested=false 2024-11-11T07:31:30,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:30,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T07:31:30,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:30,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:30,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:30,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:30,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:30,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:30,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/12a5617092e04bcf9dcabe618594c776 is 50, key is test_row_0/A:col10/1731310290610/Put/seqid=0 2024-11-11T07:31:30,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742407_1583 (size=12301) 2024-11-11T07:31:30,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T07:31:30,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:30,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:30,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:30,828 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:30,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:30,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:30,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T07:31:30,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310350838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310350840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310350841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,862 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310350844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,868 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310350854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,876 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,922 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/ac0e0a0f286f43c3980489b3a8adab3e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ac0e0a0f286f43c3980489b3a8adab3e 2024-11-11T07:31:30,942 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 752742eb661c603f835a96396e7673f2/B of 752742eb661c603f835a96396e7673f2 into ac0e0a0f286f43c3980489b3a8adab3e(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:30,942 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:30,942 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/B, priority=12, startTime=1731310290410; duration=0sec 2024-11-11T07:31:30,942 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:30,942 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:B 2024-11-11T07:31:30,953 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/6615c9d8947f48b7ae8d1fe36c248158 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/6615c9d8947f48b7ae8d1fe36c248158 2024-11-11T07:31:30,960 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 752742eb661c603f835a96396e7673f2/C of 752742eb661c603f835a96396e7673f2 into 6615c9d8947f48b7ae8d1fe36c248158(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:30,960 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:30,960 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/C, priority=12, startTime=1731310290411; duration=0sec 2024-11-11T07:31:30,960 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:30,960 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:C 2024-11-11T07:31:30,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310350965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,974 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310350965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310350965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,982 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T07:31:30,984 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:30,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:30,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310350975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:30,984 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:30,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:30,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310350985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:30,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:30,997 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,138 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T07:31:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:31,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:31,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:31,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:31,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T07:31:31,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310351185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310351185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,195 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,195 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310351185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,196 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310351197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310351205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,211 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,223 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/12a5617092e04bcf9dcabe618594c776 2024-11-11T07:31:31,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/33385d698e014bf9af078140eb84fd66 is 50, key is test_row_0/B:col10/1731310290610/Put/seqid=0 2024-11-11T07:31:31,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742408_1584 (size=12301) 2024-11-11T07:31:31,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/33385d698e014bf9af078140eb84fd66 2024-11-11T07:31:31,276 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/a9c85ac6d9114b7587d116c1633bb573 is 50, key is test_row_0/C:col10/1731310290610/Put/seqid=0 2024-11-11T07:31:31,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T07:31:31,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:31,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:31,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:31,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:31,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:31,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:31,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742409_1585 (size=12301) 2024-11-11T07:31:31,323 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=346 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/a9c85ac6d9114b7587d116c1633bb573 2024-11-11T07:31:31,327 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/12a5617092e04bcf9dcabe618594c776 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/12a5617092e04bcf9dcabe618594c776 2024-11-11T07:31:31,334 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/12a5617092e04bcf9dcabe618594c776, entries=150, sequenceid=346, filesize=12.0 K 2024-11-11T07:31:31,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/33385d698e014bf9af078140eb84fd66 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/33385d698e014bf9af078140eb84fd66 2024-11-11T07:31:31,341 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/33385d698e014bf9af078140eb84fd66, entries=150, sequenceid=346, filesize=12.0 K 2024-11-11T07:31:31,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/a9c85ac6d9114b7587d116c1633bb573 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a9c85ac6d9114b7587d116c1633bb573 2024-11-11T07:31:31,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a9c85ac6d9114b7587d116c1633bb573, entries=150, sequenceid=346, filesize=12.0 K 2024-11-11T07:31:31,352 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 752742eb661c603f835a96396e7673f2 in 592ms, sequenceid=346, compaction requested=true 2024-11-11T07:31:31,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:31,353 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:31,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:31,353 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:31,353 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:31,354 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40059 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:31,354 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/A is initiating minor compaction (all files) 2024-11-11T07:31:31,354 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/A in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:31,354 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/d8d7f7625858481b9dbc61c535307092, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/bf4cda33e673490dac8fd4df1fc724b6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/12a5617092e04bcf9dcabe618594c776] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=39.1 K 2024-11-11T07:31:31,354 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8d7f7625858481b9dbc61c535307092, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731310289820 2024-11-11T07:31:31,354 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:31,354 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/B is initiating minor compaction (all files) 2024-11-11T07:31:31,355 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/B in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:31,355 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ac0e0a0f286f43c3980489b3a8adab3e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/924b8a2fb34e46069272a526909d7eca, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/33385d698e014bf9af078140eb84fd66] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=36.7 K 2024-11-11T07:31:31,355 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting ac0e0a0f286f43c3980489b3a8adab3e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731310289820 2024-11-11T07:31:31,355 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf4cda33e673490dac8fd4df1fc724b6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731310289943 2024-11-11T07:31:31,355 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 924b8a2fb34e46069272a526909d7eca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731310289952 2024-11-11T07:31:31,355 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 33385d698e014bf9af078140eb84fd66, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731310290610 2024-11-11T07:31:31,356 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12a5617092e04bcf9dcabe618594c776, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731310290610 2024-11-11T07:31:31,355 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:31,357 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:31,359 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:31,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:31,368 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#A#compaction#499 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:31,368 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/3b78f7e1ea8449b39ce3e8bdfaec75b0 is 50, key is test_row_0/A:col10/1731310290610/Put/seqid=0 2024-11-11T07:31:31,372 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#B#compaction#500 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:31,372 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/fc4353e762114f9ca1eabec3de4a2ff1 is 50, key is test_row_0/B:col10/1731310290610/Put/seqid=0 2024-11-11T07:31:31,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742410_1586 (size=13119) 2024-11-11T07:31:31,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742411_1587 (size=13119) 2024-11-11T07:31:31,412 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/fc4353e762114f9ca1eabec3de4a2ff1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/fc4353e762114f9ca1eabec3de4a2ff1 2024-11-11T07:31:31,417 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/B of 752742eb661c603f835a96396e7673f2 into fc4353e762114f9ca1eabec3de4a2ff1(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:31,417 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:31,417 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/B, priority=13, startTime=1731310291353; duration=0sec 2024-11-11T07:31:31,417 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:31,417 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:B 2024-11-11T07:31:31,417 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:31,418 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:31,418 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/C is initiating minor compaction (all files) 2024-11-11T07:31:31,418 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/C in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:31,418 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/6615c9d8947f48b7ae8d1fe36c248158, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a9ce33ca96ba43fdb9dee59d214b8b97, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a9c85ac6d9114b7587d116c1633bb573] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=36.7 K 2024-11-11T07:31:31,419 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 6615c9d8947f48b7ae8d1fe36c248158, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731310289820 2024-11-11T07:31:31,419 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting a9ce33ca96ba43fdb9dee59d214b8b97, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1731310289952 2024-11-11T07:31:31,419 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting a9c85ac6d9114b7587d116c1633bb573, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731310290610 2024-11-11T07:31:31,430 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#C#compaction#501 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:31,430 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/666f4b4bbbef43ec9901eb5c106454af is 50, key is test_row_0/C:col10/1731310290610/Put/seqid=0 2024-11-11T07:31:31,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-11T07:31:31,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:31,454 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:31:31,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:31,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:31,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:31,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:31,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:31,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:31,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742412_1588 (size=13119) 2024-11-11T07:31:31,476 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/654d3041dc21490ba95c5793cf87e476 is 50, key is test_row_0/A:col10/1731310290839/Put/seqid=0 2024-11-11T07:31:31,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:31,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:31,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742413_1589 (size=12301) 2024-11-11T07:31:31,525 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/654d3041dc21490ba95c5793cf87e476 2024-11-11T07:31:31,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/209557f5541e4330999cda1b45f33942 is 50, key is test_row_0/B:col10/1731310290839/Put/seqid=0 2024-11-11T07:31:31,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310351539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310351541, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310351543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,566 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310351544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742414_1590 (size=12301) 2024-11-11T07:31:31,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310351562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,579 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,580 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T07:31:31,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310351665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310351675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,686 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310351675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,686 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,687 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310351675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,687 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,687 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310351688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,795 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/3b78f7e1ea8449b39ce3e8bdfaec75b0 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/3b78f7e1ea8449b39ce3e8bdfaec75b0 2024-11-11T07:31:31,799 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/A of 752742eb661c603f835a96396e7673f2 into 3b78f7e1ea8449b39ce3e8bdfaec75b0(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:31,799 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:31,799 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/A, priority=13, startTime=1731310291352; duration=0sec 2024-11-11T07:31:31,799 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:31,799 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:31,881 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/666f4b4bbbef43ec9901eb5c106454af as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/666f4b4bbbef43ec9901eb5c106454af 2024-11-11T07:31:31,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310351885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,888 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,892 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/C of 752742eb661c603f835a96396e7673f2 into 666f4b4bbbef43ec9901eb5c106454af(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:31,893 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:31,893 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/C, priority=13, startTime=1731310291358; duration=0sec 2024-11-11T07:31:31,893 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:31,893 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:C 2024-11-11T07:31:31,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310351895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310351895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310351895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,904 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,905 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:31,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310351905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:31,914 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:31,969 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/209557f5541e4330999cda1b45f33942 2024-11-11T07:31:32,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/21a9969ee05748388005338bd6ee1dd5 is 50, key is test_row_0/C:col10/1731310290839/Put/seqid=0 2024-11-11T07:31:32,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742415_1591 (size=12301) 2024-11-11T07:31:32,048 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/21a9969ee05748388005338bd6ee1dd5 2024-11-11T07:31:32,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/654d3041dc21490ba95c5793cf87e476 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/654d3041dc21490ba95c5793cf87e476 2024-11-11T07:31:32,057 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/654d3041dc21490ba95c5793cf87e476, entries=150, sequenceid=369, filesize=12.0 K 2024-11-11T07:31:32,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/209557f5541e4330999cda1b45f33942 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/209557f5541e4330999cda1b45f33942 2024-11-11T07:31:32,061 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/209557f5541e4330999cda1b45f33942, entries=150, sequenceid=369, filesize=12.0 K 2024-11-11T07:31:32,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/21a9969ee05748388005338bd6ee1dd5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/21a9969ee05748388005338bd6ee1dd5 2024-11-11T07:31:32,066 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/21a9969ee05748388005338bd6ee1dd5, entries=150, sequenceid=369, filesize=12.0 K 2024-11-11T07:31:32,067 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 752742eb661c603f835a96396e7673f2 in 613ms, sequenceid=369, compaction requested=false 2024-11-11T07:31:32,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:32,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:32,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-11T07:31:32,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-11T07:31:32,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-11T07:31:32,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5500 sec 2024-11-11T07:31:32,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 1.5550 sec 2024-11-11T07:31:32,200 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T07:31:32,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:32,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:32,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:32,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:32,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:32,200 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:32,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:32,206 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/aa357d0db7f8462587b9d2bdc443d22e is 50, key is test_row_0/A:col10/1731310292198/Put/seqid=0 2024-11-11T07:31:32,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742416_1592 (size=14741) 2024-11-11T07:31:32,213 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/aa357d0db7f8462587b9d2bdc443d22e 2024-11-11T07:31:32,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/4011a264343143bdbfe3dd161e24207b is 50, key is test_row_0/B:col10/1731310292198/Put/seqid=0 2024-11-11T07:31:32,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742417_1593 (size=12301) 2024-11-11T07:31:32,289 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:32,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:32,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310352268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:32,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310352269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:32,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310352267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:32,290 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:32,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:32,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310352271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,297 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:32,306 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:32,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310352289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,309 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:32,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310352395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:32,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:32,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310352395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:32,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:32,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310352396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:32,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,398 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:32,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:32,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:32,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:32,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310352405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,416 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:32,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:32,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310352416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,423 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:32,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:32,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310352607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,620 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:32,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:32,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310352607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:32,621 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:32,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310352607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,621 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:32,626 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,627 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:32,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:32,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310352627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:32,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,644 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:32,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:32,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310352634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:32,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:32,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:32,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-11T07:31:32,665 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:32,666 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:32,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-11T07:31:32,668 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:32,669 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:32,669 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:32,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T07:31:32,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/4011a264343143bdbfe3dd161e24207b 2024-11-11T07:31:32,697 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/195db8975fda4b44a118f5d4229c55a5 is 50, key is test_row_0/C:col10/1731310292198/Put/seqid=0 2024-11-11T07:31:32,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742418_1594 (size=12301) 2024-11-11T07:31:32,701 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/195db8975fda4b44a118f5d4229c55a5 2024-11-11T07:31:32,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/aa357d0db7f8462587b9d2bdc443d22e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/aa357d0db7f8462587b9d2bdc443d22e 2024-11-11T07:31:32,708 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/aa357d0db7f8462587b9d2bdc443d22e, entries=200, sequenceid=384, filesize=14.4 K 2024-11-11T07:31:32,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/4011a264343143bdbfe3dd161e24207b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/4011a264343143bdbfe3dd161e24207b 2024-11-11T07:31:32,713 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/4011a264343143bdbfe3dd161e24207b, entries=150, sequenceid=384, filesize=12.0 K 2024-11-11T07:31:32,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/195db8975fda4b44a118f5d4229c55a5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/195db8975fda4b44a118f5d4229c55a5 2024-11-11T07:31:32,716 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/195db8975fda4b44a118f5d4229c55a5, entries=150, sequenceid=384, filesize=12.0 K 2024-11-11T07:31:32,717 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 752742eb661c603f835a96396e7673f2 in 517ms, sequenceid=384, compaction requested=true 2024-11-11T07:31:32,717 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:32,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:32,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:32,717 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:32,717 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:32,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:32,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:32,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:32,718 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:32,718 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:32,718 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/B is initiating minor compaction (all files) 2024-11-11T07:31:32,718 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/B in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:32,718 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/fc4353e762114f9ca1eabec3de4a2ff1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/209557f5541e4330999cda1b45f33942, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/4011a264343143bdbfe3dd161e24207b] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=36.8 K 2024-11-11T07:31:32,718 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:32,719 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/A is initiating minor compaction (all files) 2024-11-11T07:31:32,719 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/A in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:32,719 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/3b78f7e1ea8449b39ce3e8bdfaec75b0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/654d3041dc21490ba95c5793cf87e476, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/aa357d0db7f8462587b9d2bdc443d22e] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=39.2 K 2024-11-11T07:31:32,720 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b78f7e1ea8449b39ce3e8bdfaec75b0, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731310290610 2024-11-11T07:31:32,720 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting fc4353e762114f9ca1eabec3de4a2ff1, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731310290610 2024-11-11T07:31:32,720 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 209557f5541e4330999cda1b45f33942, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731310290839 2024-11-11T07:31:32,720 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 654d3041dc21490ba95c5793cf87e476, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731310290839 2024-11-11T07:31:32,721 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 4011a264343143bdbfe3dd161e24207b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731310291515 2024-11-11T07:31:32,721 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa357d0db7f8462587b9d2bdc443d22e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731310291515 2024-11-11T07:31:32,727 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#B#compaction#508 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:32,727 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/7724d25f256449aa9c98fa216851cde9 is 50, key is test_row_0/B:col10/1731310292198/Put/seqid=0 2024-11-11T07:31:32,730 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#A#compaction#509 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:32,730 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/bcf4a79854dc4cdbab07d1c9998aacdd is 50, key is test_row_0/A:col10/1731310292198/Put/seqid=0 2024-11-11T07:31:32,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742420_1596 (size=13221) 2024-11-11T07:31:32,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742419_1595 (size=13221) 2024-11-11T07:31:32,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T07:31:32,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-11T07:31:32,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:32,825 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T07:31:32,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:32,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:32,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:32,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:32,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:32,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:32,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/90b9e08da1e8403a88c8930cbf8ac048 is 50, key is test_row_0/A:col10/1731310292287/Put/seqid=0 2024-11-11T07:31:32,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742421_1597 (size=12301) 2024-11-11T07:31:32,847 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/90b9e08da1e8403a88c8930cbf8ac048 2024-11-11T07:31:32,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/74ac5d4531f84ce4a746aa317d91703b is 50, key is test_row_0/B:col10/1731310292287/Put/seqid=0 2024-11-11T07:31:32,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742422_1598 (size=12301) 2024-11-11T07:31:32,898 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/74ac5d4531f84ce4a746aa317d91703b 2024-11-11T07:31:32,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/96a8ffc9a153448d9642ea7670ae144d is 50, key is test_row_0/C:col10/1731310292287/Put/seqid=0 2024-11-11T07:31:32,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:32,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:32,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742423_1599 (size=12301) 2024-11-11T07:31:32,945 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/96a8ffc9a153448d9642ea7670ae144d 2024-11-11T07:31:32,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/90b9e08da1e8403a88c8930cbf8ac048 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/90b9e08da1e8403a88c8930cbf8ac048 2024-11-11T07:31:32,952 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/90b9e08da1e8403a88c8930cbf8ac048, entries=150, sequenceid=403, filesize=12.0 K 2024-11-11T07:31:32,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/74ac5d4531f84ce4a746aa317d91703b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/74ac5d4531f84ce4a746aa317d91703b 2024-11-11T07:31:32,962 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/74ac5d4531f84ce4a746aa317d91703b, entries=150, sequenceid=403, filesize=12.0 K 2024-11-11T07:31:32,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/96a8ffc9a153448d9642ea7670ae144d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/96a8ffc9a153448d9642ea7670ae144d 2024-11-11T07:31:32,967 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/96a8ffc9a153448d9642ea7670ae144d, entries=150, sequenceid=403, filesize=12.0 K 2024-11-11T07:31:32,971 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=53.67 KB/54960 for 752742eb661c603f835a96396e7673f2 in 146ms, sequenceid=403, compaction requested=true 2024-11-11T07:31:32,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:32,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:32,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-11T07:31:32,971 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T07:31:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-11T07:31:32,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:32,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:32,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:32,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:32,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:32,972 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:32,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:32,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-11T07:31:32,981 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 307 msec 2024-11-11T07:31:32,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 317 msec 2024-11-11T07:31:32,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/4ba7e3c7735a4cdba9ef0b9a78f90ed8 is 50, key is test_row_0/A:col10/1731310292968/Put/seqid=0 2024-11-11T07:31:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-11T07:31:32,995 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:32,996 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:32,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees 2024-11-11T07:31:32,998 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:32,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T07:31:32,999 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=139, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:32,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:33,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742424_1600 (size=12297) 2024-11-11T07:31:33,018 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/4ba7e3c7735a4cdba9ef0b9a78f90ed8 2024-11-11T07:31:33,028 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/ea439d3883d34d1081933f8b52171b69 is 50, key is test_row_0/B:col10/1731310292968/Put/seqid=0 2024-11-11T07:31:33,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731310292998, max timestamp=1731310293030Num uniques 859; 2024-11-11T07:31:33,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731310292998, max timestamp=1731310293030Num uniques 859; 2024-11-11T07:31:33,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731310292998, max timestamp=1731310293030Num uniques 859; 2024-11-11T07:31:33,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T07:31:33,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T07:31:33,033 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T07:31:33,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742425_1601 (size=9857) 2024-11-11T07:31:33,043 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/ea439d3883d34d1081933f8b52171b69 2024-11-11T07:31:33,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/ec3392ee52f64f0684d8ffbe88c494a8 is 50, key is test_row_0/C:col10/1731310292968/Put/seqid=0 2024-11-11T07:31:33,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742426_1602 (size=9857) 2024-11-11T07:31:33,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/ec3392ee52f64f0684d8ffbe88c494a8 2024-11-11T07:31:33,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/4ba7e3c7735a4cdba9ef0b9a78f90ed8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/4ba7e3c7735a4cdba9ef0b9a78f90ed8 2024-11-11T07:31:33,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/4ba7e3c7735a4cdba9ef0b9a78f90ed8, entries=150, sequenceid=414, filesize=12.0 K 2024-11-11T07:31:33,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/ea439d3883d34d1081933f8b52171b69 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ea439d3883d34d1081933f8b52171b69 2024-11-11T07:31:33,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ea439d3883d34d1081933f8b52171b69, entries=100, sequenceid=414, filesize=9.6 K 2024-11-11T07:31:33,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/ec3392ee52f64f0684d8ffbe88c494a8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ec3392ee52f64f0684d8ffbe88c494a8 2024-11-11T07:31:33,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310353076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310353080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,102 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,103 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,103 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ec3392ee52f64f0684d8ffbe88c494a8, entries=100, sequenceid=414, filesize=9.6 K 2024-11-11T07:31:33,103 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 752742eb661c603f835a96396e7673f2 in 132ms, sequenceid=414, compaction requested=true 2024-11-11T07:31:33,104 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:33,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:A, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:33,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:31:33,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:B, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:33,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T07:31:33,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:33,104 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-11T07:31:33,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T07:31:33,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:33,108 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-11T07:31:33,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:33,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:33,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:33,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:33,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:33,108 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:33,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/1df37a9980b44873aa72078cca2107bf is 50, key is test_row_0/A:col10/1731310293090/Put/seqid=0 2024-11-11T07:31:33,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742427_1603 (size=14741) 2024-11-11T07:31:33,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-11T07:31:33,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:33,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:33,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:33,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:33,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:33,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:33,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310353139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310353144, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,159 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310353150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,175 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,184 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/7724d25f256449aa9c98fa216851cde9 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/7724d25f256449aa9c98fa216851cde9 2024-11-11T07:31:33,188 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/bcf4a79854dc4cdbab07d1c9998aacdd as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/bcf4a79854dc4cdbab07d1c9998aacdd 2024-11-11T07:31:33,194 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/A of 752742eb661c603f835a96396e7673f2 into bcf4a79854dc4cdbab07d1c9998aacdd(size=12.9 K), total size for store is 36.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:33,194 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:33,194 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/A, priority=13, startTime=1731310292717; duration=0sec 2024-11-11T07:31:33,194 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-11T07:31:33,194 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:33,194 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:33,195 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T07:31:33,195 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/B of 752742eb661c603f835a96396e7673f2 into 7724d25f256449aa9c98fa216851cde9(size=12.9 K), total size for store is 34.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:33,195 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:33,195 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/B, priority=13, startTime=1731310292717; duration=0sec 2024-11-11T07:31:33,195 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T07:31:33,195 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:B 2024-11-11T07:31:33,195 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:B 2024-11-11T07:31:33,195 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:33,197 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37819 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:33,197 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/A is initiating minor compaction (all files) 2024-11-11T07:31:33,197 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/A in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:33,197 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/bcf4a79854dc4cdbab07d1c9998aacdd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/90b9e08da1e8403a88c8930cbf8ac048, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/4ba7e3c7735a4cdba9ef0b9a78f90ed8] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=36.9 K 2024-11-11T07:31:33,198 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting bcf4a79854dc4cdbab07d1c9998aacdd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731310291515 2024-11-11T07:31:33,198 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 59879 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T07:31:33,198 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/C is initiating minor compaction (all files) 2024-11-11T07:31:33,198 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/C in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:33,198 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/666f4b4bbbef43ec9901eb5c106454af, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/21a9969ee05748388005338bd6ee1dd5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/195db8975fda4b44a118f5d4229c55a5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/96a8ffc9a153448d9642ea7670ae144d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ec3392ee52f64f0684d8ffbe88c494a8] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=58.5 K 2024-11-11T07:31:33,198 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 90b9e08da1e8403a88c8930cbf8ac048, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731310292267 2024-11-11T07:31:33,199 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ba7e3c7735a4cdba9ef0b9a78f90ed8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731310292944 2024-11-11T07:31:33,199 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 666f4b4bbbef43ec9901eb5c106454af, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=346, earliestPutTs=1731310290610 2024-11-11T07:31:33,199 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21a9969ee05748388005338bd6ee1dd5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1731310290839 2024-11-11T07:31:33,200 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 195db8975fda4b44a118f5d4229c55a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731310291515 2024-11-11T07:31:33,200 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96a8ffc9a153448d9642ea7670ae144d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731310292267 2024-11-11T07:31:33,208 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting ec3392ee52f64f0684d8ffbe88c494a8, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731310292944 2024-11-11T07:31:33,217 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#A#compaction#517 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:33,217 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/5528484381824ccf8af013bd16135264 is 50, key is test_row_0/A:col10/1731310292968/Put/seqid=0 2024-11-11T07:31:33,234 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310353205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310353206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,237 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#C#compaction#518 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:33,238 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/d88c075faa7a4df595eeabcb92ea6581 is 50, key is test_row_0/C:col10/1731310292968/Put/seqid=0 2024-11-11T07:31:33,238 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,239 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742429_1605 (size=13289) 2024-11-11T07:31:33,276 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/d88c075faa7a4df595eeabcb92ea6581 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/d88c075faa7a4df595eeabcb92ea6581 2024-11-11T07:31:33,279 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 752742eb661c603f835a96396e7673f2/C of 752742eb661c603f835a96396e7673f2 into d88c075faa7a4df595eeabcb92ea6581(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:33,279 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:33,279 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/C, priority=11, startTime=1731310293104; duration=0sec 2024-11-11T07:31:33,280 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:31:33,280 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:C 2024-11-11T07:31:33,280 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:33,280 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:33,280 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/B is initiating minor compaction (all files) 2024-11-11T07:31:33,280 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/B in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:33,280 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/7724d25f256449aa9c98fa216851cde9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/74ac5d4531f84ce4a746aa317d91703b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ea439d3883d34d1081933f8b52171b69] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=34.5 K 2024-11-11T07:31:33,281 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7724d25f256449aa9c98fa216851cde9, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1731310291515 2024-11-11T07:31:33,282 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74ac5d4531f84ce4a746aa317d91703b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=403, earliestPutTs=1731310292267 2024-11-11T07:31:33,282 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea439d3883d34d1081933f8b52171b69, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731310292944 2024-11-11T07:31:33,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742428_1604 (size=13323) 2024-11-11T07:31:33,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310353265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,291 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,294 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#B#compaction#519 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:33,294 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/2a0ab674bc5d4e46b8a2c0d50db41995 is 50, key is test_row_0/B:col10/1731310292968/Put/seqid=0 2024-11-11T07:31:33,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310353265, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,303 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310353285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,303 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-11T07:31:33,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:33,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:33,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:33,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:33,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:33,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:33,308 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/5528484381824ccf8af013bd16135264 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/5528484381824ccf8af013bd16135264 2024-11-11T07:31:33,312 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/A of 752742eb661c603f835a96396e7673f2 into 5528484381824ccf8af013bd16135264(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:33,312 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:33,312 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/A, priority=13, startTime=1731310293104; duration=0sec 2024-11-11T07:31:33,312 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:33,312 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:33,312 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:C 2024-11-11T07:31:33,312 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-11T07:31:33,313 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T07:31:33,313 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T07:31:33,313 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. because compaction request was cancelled 2024-11-11T07:31:33,313 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:C 2024-11-11T07:31:33,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T07:31:33,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742430_1606 (size=13323) 2024-11-11T07:31:33,360 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/2a0ab674bc5d4e46b8a2c0d50db41995 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/2a0ab674bc5d4e46b8a2c0d50db41995 2024-11-11T07:31:33,364 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/B of 752742eb661c603f835a96396e7673f2 into 2a0ab674bc5d4e46b8a2c0d50db41995(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:33,364 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:33,364 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/B, priority=13, startTime=1731310293104; duration=0sec 2024-11-11T07:31:33,364 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:33,364 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:B 2024-11-11T07:31:33,453 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310353446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,456 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310353446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-11T07:31:33,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:33,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:33,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:33,462 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] handler.RSProcedureHandler(58): pid=140 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:33,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=140 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:33,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=140 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:33,507 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310353494, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,508 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310353516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310353516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/1df37a9980b44873aa72078cca2107bf 2024-11-11T07:31:33,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/7e85068ded3045808a8ff9bc92094c73 is 50, key is test_row_0/B:col10/1731310293090/Put/seqid=0 2024-11-11T07:31:33,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742431_1607 (size=12301) 2024-11-11T07:31:33,563 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/7e85068ded3045808a8ff9bc92094c73 2024-11-11T07:31:33,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/4cd066ea33a744f2b73dabca12b0dc9f is 50, key is test_row_0/C:col10/1731310293090/Put/seqid=0 2024-11-11T07:31:33,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742432_1608 (size=12301) 2024-11-11T07:31:33,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=448 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/4cd066ea33a744f2b73dabca12b0dc9f 2024-11-11T07:31:33,584 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/1df37a9980b44873aa72078cca2107bf as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1df37a9980b44873aa72078cca2107bf 2024-11-11T07:31:33,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1df37a9980b44873aa72078cca2107bf, entries=200, sequenceid=448, filesize=14.4 K 2024-11-11T07:31:33,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/7e85068ded3045808a8ff9bc92094c73 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/7e85068ded3045808a8ff9bc92094c73 2024-11-11T07:31:33,597 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/7e85068ded3045808a8ff9bc92094c73, entries=150, sequenceid=448, filesize=12.0 K 2024-11-11T07:31:33,598 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/4cd066ea33a744f2b73dabca12b0dc9f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/4cd066ea33a744f2b73dabca12b0dc9f 2024-11-11T07:31:33,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/4cd066ea33a744f2b73dabca12b0dc9f, entries=150, sequenceid=448, filesize=12.0 K 2024-11-11T07:31:33,608 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=46.96 KB/48090 for 752742eb661c603f835a96396e7673f2 in 500ms, sequenceid=448, compaction requested=false 2024-11-11T07:31:33,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:33,622 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=140 2024-11-11T07:31:33,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:33,623 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T07:31:33,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:33,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:33,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:33,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:33,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:33,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T07:31:33,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/c7fa4f5edaed4b8298bb8e23c7f6e506 is 50, key is test_row_0/A:col10/1731310293135/Put/seqid=0 2024-11-11T07:31:33,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742433_1609 (size=12301) 2024-11-11T07:31:33,669 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/c7fa4f5edaed4b8298bb8e23c7f6e506 2024-11-11T07:31:33,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/91ca48bbe5d744db8edebff9ab555d0b is 50, key is test_row_0/B:col10/1731310293135/Put/seqid=0 2024-11-11T07:31:33,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742434_1610 (size=12301) 2024-11-11T07:31:33,727 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/91ca48bbe5d744db8edebff9ab555d0b 2024-11-11T07:31:33,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/73fcc1ef5ff64231b614e0d568ae20d8 is 50, key is test_row_0/C:col10/1731310293135/Put/seqid=0 2024-11-11T07:31:33,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:33,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:33,773 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742435_1611 (size=12301) 2024-11-11T07:31:33,774 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=463 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/73fcc1ef5ff64231b614e0d568ae20d8 2024-11-11T07:31:33,779 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/c7fa4f5edaed4b8298bb8e23c7f6e506 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c7fa4f5edaed4b8298bb8e23c7f6e506 2024-11-11T07:31:33,783 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c7fa4f5edaed4b8298bb8e23c7f6e506, entries=150, sequenceid=463, filesize=12.0 K 2024-11-11T07:31:33,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/91ca48bbe5d744db8edebff9ab555d0b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/91ca48bbe5d744db8edebff9ab555d0b 2024-11-11T07:31:33,787 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/91ca48bbe5d744db8edebff9ab555d0b, entries=150, sequenceid=463, filesize=12.0 K 2024-11-11T07:31:33,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/73fcc1ef5ff64231b614e0d568ae20d8 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/73fcc1ef5ff64231b614e0d568ae20d8 2024-11-11T07:31:33,791 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/73fcc1ef5ff64231b614e0d568ae20d8, entries=150, sequenceid=463, filesize=12.0 K 2024-11-11T07:31:33,792 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=13.42 KB/13740 for 752742eb661c603f835a96396e7673f2 in 169ms, sequenceid=463, compaction requested=true 2024-11-11T07:31:33,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:33,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:33,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-11T07:31:33,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-11-11T07:31:33,814 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-11T07:31:33,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 803 msec 2024-11-11T07:31:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:33,816 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T07:31:33,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:33,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:33,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:33,816 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:33,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:33,817 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:33,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=139, table=TestAcidGuarantees in 822 msec 2024-11-11T07:31:33,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/c5d79816126c4016b76f24138393d845 is 50, key is test_row_0/A:col10/1731310293812/Put/seqid=0 2024-11-11T07:31:33,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742436_1612 (size=12301) 2024-11-11T07:31:33,882 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/c5d79816126c4016b76f24138393d845 2024-11-11T07:31:33,890 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/458a5094678a4f72b6da3463fc252110 is 50, key is test_row_0/B:col10/1731310293812/Put/seqid=0 2024-11-11T07:31:33,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742437_1613 (size=12301) 2024-11-11T07:31:33,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731310293880, max timestamp=1731310293908Num uniques 859; 2024-11-11T07:31:33,913 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T07:31:33,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731310293880, max timestamp=1731310293908Num uniques 859; 2024-11-11T07:31:33,917 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T07:31:33,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731310293880, max timestamp=1731310293908Num uniques 859; 2024-11-11T07:31:33,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-11T07:31:33,970 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310353951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,970 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310353955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,976 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310353966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310353970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,990 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:33,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310353970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:33,991 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:33,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:34,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:34,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310354075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:34,095 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:34,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310354085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,096 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:34,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:34,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310354095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:34,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:34,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310354095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,107 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,114 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:34,114 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:34,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310354095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,115 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:34,118 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:34,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-11T07:31:34,150 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:34,150 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:34,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees 2024-11-11T07:31:34,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T07:31:34,195 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:34,196 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=141, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:34,196 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:34,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T07:31:34,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/458a5094678a4f72b6da3463fc252110 2024-11-11T07:31:34,318 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/8f1ee694842d45b09ea814fa2027aa4f is 50, key is test_row_0/C:col10/1731310293812/Put/seqid=0 2024-11-11T07:31:34,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742438_1614 (size=12301) 2024-11-11T07:31:34,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310354305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:34,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:34,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310354306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,326 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:34,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310354325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:34,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310354325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:34,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:34,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310354326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:34,342 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:34,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-11T07:31:34,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:34,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:34,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:34,359 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:34,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:34,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:34,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-11T07:31:34,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:34,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:34,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:34,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:34,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:34,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:34,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T07:31:34,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310354636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:34,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:34,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310354636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,654 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:34,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310354645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:34,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310354645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,656 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:34,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310354645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:34,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:34,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-11T07:31:34,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:34,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] handler.RSProcedureHandler(58): pid=142 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:34,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=142 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:34,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=142 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:34,724 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=473 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/8f1ee694842d45b09ea814fa2027aa4f 2024-11-11T07:31:34,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/c5d79816126c4016b76f24138393d845 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c5d79816126c4016b76f24138393d845 2024-11-11T07:31:34,738 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c5d79816126c4016b76f24138393d845, entries=150, sequenceid=473, filesize=12.0 K 2024-11-11T07:31:34,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/458a5094678a4f72b6da3463fc252110 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/458a5094678a4f72b6da3463fc252110 2024-11-11T07:31:34,746 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/458a5094678a4f72b6da3463fc252110, entries=150, sequenceid=473, filesize=12.0 K 2024-11-11T07:31:34,749 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/8f1ee694842d45b09ea814fa2027aa4f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8f1ee694842d45b09ea814fa2027aa4f 2024-11-11T07:31:34,752 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8f1ee694842d45b09ea814fa2027aa4f, entries=150, sequenceid=473, filesize=12.0 K 2024-11-11T07:31:34,761 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=147.60 KB/151140 for 752742eb661c603f835a96396e7673f2 in 945ms, sequenceid=473, compaction requested=true 2024-11-11T07:31:34,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:34,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:34,761 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:34,761 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:34,761 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:34,762 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52666 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:34,762 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/A is initiating minor compaction (all files) 2024-11-11T07:31:34,762 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/A in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:34,763 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/5528484381824ccf8af013bd16135264, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1df37a9980b44873aa72078cca2107bf, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c7fa4f5edaed4b8298bb8e23c7f6e506, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c5d79816126c4016b76f24138393d845] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=51.4 K 2024-11-11T07:31:34,763 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:34,763 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/B is initiating minor compaction (all files) 2024-11-11T07:31:34,763 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/B in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:34,763 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/2a0ab674bc5d4e46b8a2c0d50db41995, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/7e85068ded3045808a8ff9bc92094c73, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/91ca48bbe5d744db8edebff9ab555d0b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/458a5094678a4f72b6da3463fc252110] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=49.0 K 2024-11-11T07:31:34,763 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5528484381824ccf8af013bd16135264, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731310292268 2024-11-11T07:31:34,764 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a0ab674bc5d4e46b8a2c0d50db41995, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731310292268 2024-11-11T07:31:34,764 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1df37a9980b44873aa72078cca2107bf, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731310293080 2024-11-11T07:31:34,764 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e85068ded3045808a8ff9bc92094c73, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731310293080 2024-11-11T07:31:34,764 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7fa4f5edaed4b8298bb8e23c7f6e506, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731310293135 2024-11-11T07:31:34,764 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 91ca48bbe5d744db8edebff9ab555d0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731310293135 2024-11-11T07:31:34,764 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5d79816126c4016b76f24138393d845, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731310293800 2024-11-11T07:31:34,765 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 458a5094678a4f72b6da3463fc252110, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731310293800 2024-11-11T07:31:34,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:34,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:34,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:34,772 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:34,777 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#A#compaction#528 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:34,777 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/1febd04466b34b2eaa7750928e099170 is 50, key is test_row_0/A:col10/1731310293812/Put/seqid=0 2024-11-11T07:31:34,782 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#B#compaction#529 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:34,783 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/dc935753a39344e8a3c7c7d314504578 is 50, key is test_row_0/B:col10/1731310293812/Put/seqid=0 2024-11-11T07:31:34,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742439_1615 (size=13459) 2024-11-11T07:31:34,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742440_1616 (size=13459) 2024-11-11T07:31:34,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=142 2024-11-11T07:31:34,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:34,821 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=147.60 KB heapSize=406.08 KB 2024-11-11T07:31:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:34,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T07:31:34,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/c0731e8603e3404cb2c6281ba48862c3 is 50, key is test_row_0/A:col10/1731310293967/Put/seqid=0 2024-11-11T07:31:34,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742441_1617 (size=12301) 2024-11-11T07:31:35,175 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:35,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:35,185 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310355177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,186 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,194 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310355178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,195 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310355181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310355182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,198 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,200 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/1febd04466b34b2eaa7750928e099170 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1febd04466b34b2eaa7750928e099170 2024-11-11T07:31:35,203 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/dc935753a39344e8a3c7c7d314504578 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/dc935753a39344e8a3c7c7d314504578 2024-11-11T07:31:35,207 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310355186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,207 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 752742eb661c603f835a96396e7673f2/A of 752742eb661c603f835a96396e7673f2 into 1febd04466b34b2eaa7750928e099170(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:35,207 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:35,207 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/A, priority=12, startTime=1731310294761; duration=0sec 2024-11-11T07:31:35,208 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:35,208 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:35,208 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,208 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,212 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 752742eb661c603f835a96396e7673f2/B of 752742eb661c603f835a96396e7673f2 into dc935753a39344e8a3c7c7d314504578(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:35,212 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:35,212 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/B, priority=12, startTime=1731310294761; duration=0sec 2024-11-11T07:31:35,212 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:35,212 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:B 2024-11-11T07:31:35,213 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50192 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:35,213 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/C is initiating minor compaction (all files) 2024-11-11T07:31:35,213 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/C in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:35,213 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/d88c075faa7a4df595eeabcb92ea6581, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/4cd066ea33a744f2b73dabca12b0dc9f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/73fcc1ef5ff64231b614e0d568ae20d8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8f1ee694842d45b09ea814fa2027aa4f] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=49.0 K 2024-11-11T07:31:35,213 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting d88c075faa7a4df595eeabcb92ea6581, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=414, earliestPutTs=1731310292268 2024-11-11T07:31:35,214 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cd066ea33a744f2b73dabca12b0dc9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=448, earliestPutTs=1731310293080 2024-11-11T07:31:35,214 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73fcc1ef5ff64231b614e0d568ae20d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=463, earliestPutTs=1731310293135 2024-11-11T07:31:35,214 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8f1ee694842d45b09ea814fa2027aa4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731310293800 2024-11-11T07:31:35,226 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#C#compaction#531 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:35,226 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/49cb389fb2964e6d9ab5e0fa62d6d6d2 is 50, key is test_row_0/C:col10/1731310293812/Put/seqid=0 2024-11-11T07:31:35,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742442_1618 (size=13425) 2024-11-11T07:31:35,241 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/49cb389fb2964e6d9ab5e0fa62d6d6d2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/49cb389fb2964e6d9ab5e0fa62d6d6d2 2024-11-11T07:31:35,245 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 752742eb661c603f835a96396e7673f2/C of 752742eb661c603f835a96396e7673f2 into 49cb389fb2964e6d9ab5e0fa62d6d6d2(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:35,245 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:35,245 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/C, priority=12, startTime=1731310294772; duration=0sec 2024-11-11T07:31:35,245 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:35,245 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:C 2024-11-11T07:31:35,269 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/c0731e8603e3404cb2c6281ba48862c3 2024-11-11T07:31:35,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/bc1f84dd2f224d6a87df889efa9d38a7 is 50, key is test_row_0/B:col10/1731310293967/Put/seqid=0 2024-11-11T07:31:35,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310355302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310355304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310355305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310355305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,323 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,323 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,325 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310355315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T07:31:35,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742443_1619 (size=12301) 2024-11-11T07:31:35,546 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310355538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310355538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,547 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310355539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,548 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310355539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310355539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,739 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/bc1f84dd2f224d6a87df889efa9d38a7 2024-11-11T07:31:35,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/e89fda178ef54e0096f570c1f82a2d11 is 50, key is test_row_0/C:col10/1731310293967/Put/seqid=0 2024-11-11T07:31:35,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742444_1620 (size=12301) 2024-11-11T07:31:35,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310355855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310355856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310355856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,862 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310355856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,863 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:35,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310355856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:35,863 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,175 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=508 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/e89fda178ef54e0096f570c1f82a2d11 2024-11-11T07:31:36,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/c0731e8603e3404cb2c6281ba48862c3 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c0731e8603e3404cb2c6281ba48862c3 2024-11-11T07:31:36,186 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c0731e8603e3404cb2c6281ba48862c3, entries=150, sequenceid=508, filesize=12.0 K 2024-11-11T07:31:36,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/bc1f84dd2f224d6a87df889efa9d38a7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/bc1f84dd2f224d6a87df889efa9d38a7 2024-11-11T07:31:36,191 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/bc1f84dd2f224d6a87df889efa9d38a7, entries=150, sequenceid=508, filesize=12.0 K 2024-11-11T07:31:36,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/e89fda178ef54e0096f570c1f82a2d11 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/e89fda178ef54e0096f570c1f82a2d11 2024-11-11T07:31:36,202 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/e89fda178ef54e0096f570c1f82a2d11, entries=150, sequenceid=508, filesize=12.0 K 2024-11-11T07:31:36,203 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~406.03 KB/415776, currentSize=40.25 KB/41220 for 752742eb661c603f835a96396e7673f2 in 1382ms, sequenceid=508, compaction requested=false 2024-11-11T07:31:36,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:36,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:36,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-11T07:31:36,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-11-11T07:31:36,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-11T07:31:36,209 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0110 sec 2024-11-11T07:31:36,214 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=141, table=TestAcidGuarantees in 2.0590 sec 2024-11-11T07:31:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-11T07:31:36,344 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:36,345 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:36,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees 2024-11-11T07:31:36,347 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:36,348 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=143, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:36,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:36,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-11T07:31:36,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T07:31:36,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:36,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:36,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:36,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:36,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:36,381 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:36,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:36,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/64e47e2c19ec46d8ada549e5e0fc2f80 is 50, key is test_row_0/A:col10/1731310295182/Put/seqid=0 2024-11-11T07:31:36,408 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742445_1621 (size=12297) 2024-11-11T07:31:36,409 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/64e47e2c19ec46d8ada549e5e0fc2f80 2024-11-11T07:31:36,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/0a6368cb880c4b58a464895555759865 is 50, key is test_row_0/B:col10/1731310295182/Put/seqid=0 2024-11-11T07:31:36,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-11T07:31:36,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742446_1622 (size=9857) 2024-11-11T07:31:36,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310356440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,460 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,460 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310356449, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,461 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310356451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,461 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731310296400, max timestamp=1731310296454Num uniques 859; 2024-11-11T07:31:36,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731310296400, max timestamp=1731310296456Num uniques 859; 2024-11-11T07:31:36,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T07:31:36,464 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T07:31:36,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731310296400, max timestamp=1731310296454Num uniques 859; 2024-11-11T07:31:36,466 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T07:31:36,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-11T07:31:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:36,507 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:36,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:36,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:36,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310356590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,599 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310356593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,615 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310356594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310356594, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,616 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,616 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310356595, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-11T07:31:36,666 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-11T07:31:36,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:36,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:36,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:36,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:36,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:36,710 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310356704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,711 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310356727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,728 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310356727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310356727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310356727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,731 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-11T07:31:36,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:36,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:36,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:36,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:36,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:36,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:36,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/0a6368cb880c4b58a464895555759865 2024-11-11T07:31:36,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/fff7b4e04b02431d888a09f54cb7a27b is 50, key is test_row_0/C:col10/1731310295182/Put/seqid=0 2024-11-11T07:31:36,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742447_1623 (size=9857) 2024-11-11T07:31:36,885 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=521 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/fff7b4e04b02431d888a09f54cb7a27b 2024-11-11T07:31:36,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/64e47e2c19ec46d8ada549e5e0fc2f80 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/64e47e2c19ec46d8ada549e5e0fc2f80 2024-11-11T07:31:36,895 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/64e47e2c19ec46d8ada549e5e0fc2f80, entries=150, sequenceid=521, filesize=12.0 K 2024-11-11T07:31:36,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/0a6368cb880c4b58a464895555759865 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0a6368cb880c4b58a464895555759865 2024-11-11T07:31:36,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0a6368cb880c4b58a464895555759865, entries=100, sequenceid=521, filesize=9.6 K 2024-11-11T07:31:36,900 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/fff7b4e04b02431d888a09f54cb7a27b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/fff7b4e04b02431d888a09f54cb7a27b 2024-11-11T07:31:36,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/fff7b4e04b02431d888a09f54cb7a27b, entries=100, sequenceid=521, filesize=9.6 K 2024-11-11T07:31:36,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=132.99 KB/136185 for 752742eb661c603f835a96396e7673f2 in 527ms, sequenceid=521, compaction requested=true 2024-11-11T07:31:36,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:36,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:36,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:36,907 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:36,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:36,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:36,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:36,907 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:36,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:36,908 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38057 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:36,908 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/A is initiating minor compaction (all files) 2024-11-11T07:31:36,908 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/A in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:36,908 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1febd04466b34b2eaa7750928e099170, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c0731e8603e3404cb2c6281ba48862c3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/64e47e2c19ec46d8ada549e5e0fc2f80] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=37.2 K 2024-11-11T07:31:36,911 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35617 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:36,911 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/B is initiating minor compaction (all files) 2024-11-11T07:31:36,911 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/B in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:36,911 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/dc935753a39344e8a3c7c7d314504578, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/bc1f84dd2f224d6a87df889efa9d38a7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0a6368cb880c4b58a464895555759865] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=34.8 K 2024-11-11T07:31:36,912 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1febd04466b34b2eaa7750928e099170, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731310293800 2024-11-11T07:31:36,912 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting dc935753a39344e8a3c7c7d314504578, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731310293800 2024-11-11T07:31:36,912 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0731e8603e3404cb2c6281ba48862c3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731310293940 2024-11-11T07:31:36,912 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting bc1f84dd2f224d6a87df889efa9d38a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731310293940 2024-11-11T07:31:36,912 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a6368cb880c4b58a464895555759865, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731310295182 2024-11-11T07:31:36,913 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 64e47e2c19ec46d8ada549e5e0fc2f80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731310295177 2024-11-11T07:31:36,920 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#B#compaction#537 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:36,921 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/795b39e705d94160981d6dadc41c6cf7 is 50, key is test_row_0/B:col10/1731310295182/Put/seqid=0 2024-11-11T07:31:36,924 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#A#compaction#538 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:36,925 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/bead7466e5b84629b7a1a8b2a8a9378b is 50, key is test_row_0/A:col10/1731310295182/Put/seqid=0 2024-11-11T07:31:36,930 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=139.70 KB heapSize=402.87 KB 2024-11-11T07:31:36,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:36,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T07:31:36,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:36,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T07:31:36,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:36,931 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T07:31:36,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:36,965 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/046e6771cb68442cbeda176316c0ab29 is 50, key is test_row_0/A:col10/1731310296929/Put/seqid=0 2024-11-11T07:31:36,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-11T07:31:36,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310356957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,978 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310356958, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310356962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-11T07:31:36,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:36,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:36,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:36,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:36,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:36,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,982 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,983 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:36,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742449_1625 (size=13561) 2024-11-11T07:31:36,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:36,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310356978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:36,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:36,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310356979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742448_1624 (size=13561) 2024-11-11T07:31:37,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742450_1626 (size=14741) 2024-11-11T07:31:37,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/046e6771cb68442cbeda176316c0ab29 2024-11-11T07:31:37,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/153d8fd221ec4896805a5fbba50a1d35 is 50, key is test_row_0/B:col10/1731310296929/Put/seqid=0 2024-11-11T07:31:37,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742451_1627 (size=12301) 2024-11-11T07:31:37,040 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/153d8fd221ec4896805a5fbba50a1d35 2024-11-11T07:31:37,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/a2f91c438f524db7a34948669dab1704 is 50, key is test_row_0/C:col10/1731310296929/Put/seqid=0 2024-11-11T07:31:37,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742452_1628 (size=12301) 2024-11-11T07:31:37,072 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=559 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/a2f91c438f524db7a34948669dab1704 2024-11-11T07:31:37,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/046e6771cb68442cbeda176316c0ab29 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/046e6771cb68442cbeda176316c0ab29 2024-11-11T07:31:37,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/046e6771cb68442cbeda176316c0ab29, entries=200, sequenceid=559, filesize=14.4 K 2024-11-11T07:31:37,083 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/153d8fd221ec4896805a5fbba50a1d35 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/153d8fd221ec4896805a5fbba50a1d35 2024-11-11T07:31:37,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/153d8fd221ec4896805a5fbba50a1d35, entries=150, sequenceid=559, filesize=12.0 K 2024-11-11T07:31:37,087 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/a2f91c438f524db7a34948669dab1704 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a2f91c438f524db7a34948669dab1704 2024-11-11T07:31:37,091 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a2f91c438f524db7a34948669dab1704, entries=150, sequenceid=559, filesize=12.0 K 2024-11-11T07:31:37,092 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~139.70 KB/143055, heapSize ~402.82 KB/412488, currentSize=46.96 KB/48090 for 752742eb661c603f835a96396e7673f2 in 162ms, sequenceid=559, compaction requested=true 2024-11-11T07:31:37,092 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:37,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:A, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:37,092 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:31:37,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:B, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:37,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T07:31:37,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:37,093 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-11T07:31:37,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:37,096 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T07:31:37,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:37,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:37,097 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:37,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:37,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:37,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:37,108 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/501abd5815ad41deac140fb01d63e936 is 50, key is test_row_0/A:col10/1731310296976/Put/seqid=0 2024-11-11T07:31:37,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-11T07:31:37,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:37,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:37,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:37,134 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] handler.RSProcedureHandler(58): pid=144 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:37,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=144 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=144 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:37,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742453_1629 (size=17181) 2024-11-11T07:31:37,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/501abd5815ad41deac140fb01d63e936 2024-11-11T07:31:37,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/57d1ce4efc694a60badf99a588a6b5a5 is 50, key is test_row_0/B:col10/1731310296976/Put/seqid=0 2024-11-11T07:31:37,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742454_1630 (size=12301) 2024-11-11T07:31:37,181 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/57d1ce4efc694a60badf99a588a6b5a5 2024-11-11T07:31:37,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310357164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,182 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,183 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,188 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/2215253d59f64b08b087785ee815490d is 50, key is test_row_0/C:col10/1731310296976/Put/seqid=0 2024-11-11T07:31:37,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310357172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742455_1631 (size=12301) 2024-11-11T07:31:37,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310357174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,193 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,193 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=571 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/2215253d59f64b08b087785ee815490d 2024-11-11T07:31:37,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310357181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,201 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/501abd5815ad41deac140fb01d63e936 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/501abd5815ad41deac140fb01d63e936 2024-11-11T07:31:37,204 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/501abd5815ad41deac140fb01d63e936, entries=250, sequenceid=571, filesize=16.8 K 2024-11-11T07:31:37,205 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/57d1ce4efc694a60badf99a588a6b5a5 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/57d1ce4efc694a60badf99a588a6b5a5 2024-11-11T07:31:37,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/57d1ce4efc694a60badf99a588a6b5a5, entries=150, sequenceid=571, filesize=12.0 K 2024-11-11T07:31:37,210 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/2215253d59f64b08b087785ee815490d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/2215253d59f64b08b087785ee815490d 2024-11-11T07:31:37,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310357183, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,217 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/2215253d59f64b08b087785ee815490d, entries=150, sequenceid=571, filesize=12.0 K 2024-11-11T07:31:37,231 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 752742eb661c603f835a96396e7673f2 in 135ms, sequenceid=571, compaction requested=true 2024-11-11T07:31:37,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:37,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:A, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:37,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2024-11-11T07:31:37,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:B, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:37,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=6), splitQueue=0 2024-11-11T07:31:37,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:37,231 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2024-11-11T07:31:37,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=144 2024-11-11T07:31:37,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:37,289 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:31:37,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:37,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:37,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:37,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:37,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:37,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:37,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:37,301 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:37,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/f34b18e72b01479698b4a26a45ff8d55 is 50, key is test_row_0/A:col10/1731310297168/Put/seqid=0 2024-11-11T07:31:37,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310357329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,358 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310357331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,358 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742456_1632 (size=12301) 2024-11-11T07:31:37,360 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/f34b18e72b01479698b4a26a45ff8d55 2024-11-11T07:31:37,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/4c8c6244893240cc98c75efdb622a832 is 50, key is test_row_0/B:col10/1731310297168/Put/seqid=0 2024-11-11T07:31:37,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742457_1633 (size=12301) 2024-11-11T07:31:37,377 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/4c8c6244893240cc98c75efdb622a832 2024-11-11T07:31:37,384 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310357354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,385 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310357356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,385 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,385 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310357358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,385 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,386 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/9765381606af49bfbb851caec09af9d9 is 50, key is test_row_0/C:col10/1731310297168/Put/seqid=0 2024-11-11T07:31:37,396 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/bead7466e5b84629b7a1a8b2a8a9378b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/bead7466e5b84629b7a1a8b2a8a9378b 2024-11-11T07:31:37,401 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/A of 752742eb661c603f835a96396e7673f2 into bead7466e5b84629b7a1a8b2a8a9378b(size=13.2 K), total size for store is 44.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:37,401 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:37,401 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/A, priority=13, startTime=1731310296907; duration=0sec 2024-11-11T07:31:37,401 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=7), splitQueue=0 2024-11-11T07:31:37,401 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:37,401 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:37,401 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-11T07:31:37,402 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60185 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-11T07:31:37,402 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/C is initiating minor compaction (all files) 2024-11-11T07:31:37,403 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/C in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:37,403 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/49cb389fb2964e6d9ab5e0fa62d6d6d2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/e89fda178ef54e0096f570c1f82a2d11, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/fff7b4e04b02431d888a09f54cb7a27b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a2f91c438f524db7a34948669dab1704, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/2215253d59f64b08b087785ee815490d] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=58.8 K 2024-11-11T07:31:37,404 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49cb389fb2964e6d9ab5e0fa62d6d6d2, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=473, earliestPutTs=1731310293800 2024-11-11T07:31:37,404 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting e89fda178ef54e0096f570c1f82a2d11, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=508, earliestPutTs=1731310293940 2024-11-11T07:31:37,404 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting fff7b4e04b02431d888a09f54cb7a27b, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731310295182 2024-11-11T07:31:37,404 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2f91c438f524db7a34948669dab1704, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731310296589 2024-11-11T07:31:37,404 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2215253d59f64b08b087785ee815490d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731310296976 2024-11-11T07:31:37,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742458_1634 (size=12301) 2024-11-11T07:31:37,416 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#C#compaction#548 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:37,417 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/9bad78bbae5242938a770e5f4bea22a4 is 50, key is test_row_0/C:col10/1731310296976/Put/seqid=0 2024-11-11T07:31:37,418 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/9765381606af49bfbb851caec09af9d9 2024-11-11T07:31:37,423 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/795b39e705d94160981d6dadc41c6cf7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/795b39e705d94160981d6dadc41c6cf7 2024-11-11T07:31:37,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/f34b18e72b01479698b4a26a45ff8d55 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/f34b18e72b01479698b4a26a45ff8d55 2024-11-11T07:31:37,429 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/B of 752742eb661c603f835a96396e7673f2 into 795b39e705d94160981d6dadc41c6cf7(size=13.2 K), total size for store is 37.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:37,429 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:37,429 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/B, priority=13, startTime=1731310296907; duration=0sec 2024-11-11T07:31:37,429 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=6), splitQueue=0 2024-11-11T07:31:37,429 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:B 2024-11-11T07:31:37,429 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/f34b18e72b01479698b4a26a45ff8d55, entries=150, sequenceid=591, filesize=12.0 K 2024-11-11T07:31:37,429 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:B 2024-11-11T07:31:37,429 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:37,433 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 57784 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:37,433 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/A is initiating minor compaction (all files) 2024-11-11T07:31:37,433 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/A in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:37,433 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/bead7466e5b84629b7a1a8b2a8a9378b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/046e6771cb68442cbeda176316c0ab29, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/501abd5815ad41deac140fb01d63e936, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/f34b18e72b01479698b4a26a45ff8d55] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=56.4 K 2024-11-11T07:31:37,437 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting bead7466e5b84629b7a1a8b2a8a9378b, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731310293940 2024-11-11T07:31:37,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/4c8c6244893240cc98c75efdb622a832 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/4c8c6244893240cc98c75efdb622a832 2024-11-11T07:31:37,437 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 046e6771cb68442cbeda176316c0ab29, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731310296589 2024-11-11T07:31:37,438 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 501abd5815ad41deac140fb01d63e936, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731310296956 2024-11-11T07:31:37,438 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f34b18e72b01479698b4a26a45ff8d55, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731310297156 2024-11-11T07:31:37,441 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/4c8c6244893240cc98c75efdb622a832, entries=150, sequenceid=591, filesize=12.0 K 2024-11-11T07:31:37,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/9765381606af49bfbb851caec09af9d9 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/9765381606af49bfbb851caec09af9d9 2024-11-11T07:31:37,457 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/9765381606af49bfbb851caec09af9d9, entries=150, sequenceid=591, filesize=12.0 K 2024-11-11T07:31:37,458 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 752742eb661c603f835a96396e7673f2 in 169ms, sequenceid=591, compaction requested=true 2024-11-11T07:31:37,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:37,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:37,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-11T07:31:37,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-11-11T07:31:37,461 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#A#compaction#549 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:37,461 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/91553cf554674f278552b49f2cf8a334 is 50, key is test_row_0/A:col10/1731310297168/Put/seqid=0 2024-11-11T07:31:37,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-11T07:31:37,464 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1140 sec 2024-11-11T07:31:37,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742459_1635 (size=13595) 2024-11-11T07:31:37,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=143, table=TestAcidGuarantees in 1.1240 sec 2024-11-11T07:31:37,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:37,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T07:31:37,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:37,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:37,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:37,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:37,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:37,477 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:37,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-11T07:31:37,484 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:37,485 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees 2024-11-11T07:31:37,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/45d8b1383dcf437f87af84bd4137b3ef is 50, key is test_row_0/A:col10/1731310297352/Put/seqid=0 2024-11-11T07:31:37,489 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:37,490 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=145, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-11T07:31:37,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:37,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742460_1636 (size=13697) 2024-11-11T07:31:37,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742461_1637 (size=14741) 2024-11-11T07:31:37,524 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/91553cf554674f278552b49f2cf8a334 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/91553cf554674f278552b49f2cf8a334 2024-11-11T07:31:37,524 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/45d8b1383dcf437f87af84bd4137b3ef 2024-11-11T07:31:37,528 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 752742eb661c603f835a96396e7673f2/A of 752742eb661c603f835a96396e7673f2 into 91553cf554674f278552b49f2cf8a334(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:37,529 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:37,529 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/A, priority=12, startTime=1731310297092; duration=0sec 2024-11-11T07:31:37,529 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=5), splitQueue=0 2024-11-11T07:31:37,529 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:37,529 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 5 compacting, 1 eligible, 16 blocking 2024-11-11T07:31:37,531 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T07:31:37,531 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T07:31:37,531 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. because compaction request was cancelled 2024-11-11T07:31:37,531 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:C 2024-11-11T07:31:37,531 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:37,532 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:37,532 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/B is initiating minor compaction (all files) 2024-11-11T07:31:37,532 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/B in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:37,532 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/795b39e705d94160981d6dadc41c6cf7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/153d8fd221ec4896805a5fbba50a1d35, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/57d1ce4efc694a60badf99a588a6b5a5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/4c8c6244893240cc98c75efdb622a832] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=49.3 K 2024-11-11T07:31:37,533 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 795b39e705d94160981d6dadc41c6cf7, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=521, earliestPutTs=1731310293940 2024-11-11T07:31:37,533 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 153d8fd221ec4896805a5fbba50a1d35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=559, earliestPutTs=1731310296589 2024-11-11T07:31:37,533 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 57d1ce4efc694a60badf99a588a6b5a5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731310296976 2024-11-11T07:31:37,533 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 4c8c6244893240cc98c75efdb622a832, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731310297156 2024-11-11T07:31:37,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/5e4c415e93334060979879ea6c63094a is 50, key is test_row_0/B:col10/1731310297352/Put/seqid=0 2024-11-11T07:31:37,554 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#B#compaction#552 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:37,554 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/d5367e3f5511460bac5e8f2d42865a88 is 50, key is test_row_0/B:col10/1731310297168/Put/seqid=0 2024-11-11T07:31:37,556 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310357534, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310357536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,557 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310357550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310357559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,576 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310357560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742463_1639 (size=13697) 2024-11-11T07:31:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,579 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742462_1638 (size=12301) 2024-11-11T07:31:37,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-11T07:31:37,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-11T07:31:37,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:37,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:37,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:37,648 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:37,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:37,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:37,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310357665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310357665, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310357674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310357674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310357684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,697 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-11T07:31:37,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-11T07:31:37,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:37,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:37,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:37,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:37,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:37,866 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/9bad78bbae5242938a770e5f4bea22a4 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/9bad78bbae5242938a770e5f4bea22a4 2024-11-11T07:31:37,875 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 752742eb661c603f835a96396e7673f2/C of 752742eb661c603f835a96396e7673f2 into 9bad78bbae5242938a770e5f4bea22a4(size=13.3 K), total size for store is 25.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:37,875 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:37,875 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/C, priority=11, startTime=1731310297231; duration=0sec 2024-11-11T07:31:37,875 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-11T07:31:37,875 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:C 2024-11-11T07:31:37,875 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:C 2024-11-11T07:31:37,875 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:37,875 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 4 compacting, 0 eligible, 16 blocking 2024-11-11T07:31:37,875 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T07:31:37,875 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T07:31:37,875 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. because compaction request was cancelled 2024-11-11T07:31:37,875 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:B 2024-11-11T07:31:37,875 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T07:31:37,876 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T07:31:37,876 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T07:31:37,876 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. because compaction request was cancelled 2024-11-11T07:31:37,876 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:C 2024-11-11T07:31:37,876 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-11T07:31:37,876 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T07:31:37,876 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T07:31:37,876 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. because compaction request was cancelled 2024-11-11T07:31:37,876 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:37,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310357875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310357884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,893 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310357895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310357895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310357905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:37,906 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:37,959 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-11T07:31:37,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:37,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:37,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:37,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:37,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:37,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:37,984 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/5e4c415e93334060979879ea6c63094a 2024-11-11T07:31:37,984 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/d5367e3f5511460bac5e8f2d42865a88 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/d5367e3f5511460bac5e8f2d42865a88 2024-11-11T07:31:37,992 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 752742eb661c603f835a96396e7673f2/B of 752742eb661c603f835a96396e7673f2 into d5367e3f5511460bac5e8f2d42865a88(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:37,992 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:37,992 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/B, priority=12, startTime=1731310297092; duration=0sec 2024-11-11T07:31:37,992 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:37,992 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:B 2024-11-11T07:31:37,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/5b703fc4efda481695e7d45ac6f8e81f is 50, key is test_row_0/C:col10/1731310297352/Put/seqid=0 2024-11-11T07:31:38,019 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742464_1640 (size=12301) 2024-11-11T07:31:38,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-11T07:31:38,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:38,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:38,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:38,115 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:38,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-11T07:31:38,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:38,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:38,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36262 deadline: 1731310358194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:38,204 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:38,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:38,205 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:38,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:38,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36272 deadline: 1731310358205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:38,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:38,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:38,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36228 deadline: 1731310358215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:38,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36244 deadline: 1731310358215, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:38,224 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:38,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:36304 deadline: 1731310358216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 , the old value is region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:38,226 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., hostname=ce2cf0244921,33345,1731310155369, seqNum=2 because the exception is null or not the one we care about 2024-11-11T07:31:38,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-11T07:31:38,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:38,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:38,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:38,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:38,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:38,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:38,421 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=606 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/5b703fc4efda481695e7d45ac6f8e81f 2024-11-11T07:31:38,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/45d8b1383dcf437f87af84bd4137b3ef as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/45d8b1383dcf437f87af84bd4137b3ef 2024-11-11T07:31:38,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-11T07:31:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:38,427 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] handler.RSProcedureHandler(58): pid=146 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:38,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=146 java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:38,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=146 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:38,431 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/45d8b1383dcf437f87af84bd4137b3ef, entries=200, sequenceid=606, filesize=14.4 K 2024-11-11T07:31:38,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/5e4c415e93334060979879ea6c63094a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/5e4c415e93334060979879ea6c63094a 2024-11-11T07:31:38,437 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/5e4c415e93334060979879ea6c63094a, entries=150, sequenceid=606, filesize=12.0 K 2024-11-11T07:31:38,438 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/5b703fc4efda481695e7d45ac6f8e81f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/5b703fc4efda481695e7d45ac6f8e81f 2024-11-11T07:31:38,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/5b703fc4efda481695e7d45ac6f8e81f, entries=150, sequenceid=606, filesize=12.0 K 2024-11-11T07:31:38,454 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 752742eb661c603f835a96396e7673f2 in 977ms, sequenceid=606, compaction requested=true 2024-11-11T07:31:38,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:38,454 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T07:31:38,454 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:38,455 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T07:31:38,455 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T07:31:38,455 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. because compaction request was cancelled 2024-11-11T07:31:38,455 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:A 2024-11-11T07:31:38,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:38,455 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T07:31:38,455 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T07:31:38,455 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T07:31:38,456 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. because compaction request was cancelled 2024-11-11T07:31:38,456 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:B 2024-11-11T07:31:38,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:B, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:38,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:38,456 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:38,456 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 752742eb661c603f835a96396e7673f2:C, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:38,456 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38197 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:38,456 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 752742eb661c603f835a96396e7673f2/C is initiating minor compaction (all files) 2024-11-11T07:31:38,456 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 752742eb661c603f835a96396e7673f2/C in TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:38,456 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/9bad78bbae5242938a770e5f4bea22a4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/9765381606af49bfbb851caec09af9d9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/5b703fc4efda481695e7d45ac6f8e81f] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp, totalSize=37.3 K 2024-11-11T07:31:38,457 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:38,457 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9bad78bbae5242938a770e5f4bea22a4, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=571, earliestPutTs=1731310296976 2024-11-11T07:31:38,457 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9765381606af49bfbb851caec09af9d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731310297156 2024-11-11T07:31:38,457 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b703fc4efda481695e7d45ac6f8e81f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=606, earliestPutTs=1731310297352 2024-11-11T07:31:38,464 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 752742eb661c603f835a96396e7673f2#C#compaction#554 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:38,465 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/589b35f769b8437fac9bcdfb620d4dfb is 50, key is test_row_0/C:col10/1731310297352/Put/seqid=0 2024-11-11T07:31:38,482 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742465_1641 (size=13697) 2024-11-11T07:31:38,491 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/589b35f769b8437fac9bcdfb620d4dfb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/589b35f769b8437fac9bcdfb620d4dfb 2024-11-11T07:31:38,502 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 752742eb661c603f835a96396e7673f2/C of 752742eb661c603f835a96396e7673f2 into 589b35f769b8437fac9bcdfb620d4dfb(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:38,502 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:38,502 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., storeName=752742eb661c603f835a96396e7673f2/C, priority=13, startTime=1731310298456; duration=0sec 2024-11-11T07:31:38,502 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:38,502 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 752742eb661c603f835a96396e7673f2:C 2024-11-11T07:31:38,580 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=146 2024-11-11T07:31:38,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:38,583 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T07:31:38,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:38,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:38,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:38,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:38,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:38,583 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:38,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/9cf630e8a7f2424986de3e1b6ed7d91e is 50, key is test_row_0/A:col10/1731310297551/Put/seqid=0 2024-11-11T07:31:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-11T07:31:38,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742466_1642 (size=12301) 2024-11-11T07:31:38,644 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/9cf630e8a7f2424986de3e1b6ed7d91e 2024-11-11T07:31:38,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/42baff9a8a7b40068fcc1ea12ea886aa is 50, key is test_row_0/B:col10/1731310297551/Put/seqid=0 2024-11-11T07:31:38,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742467_1643 (size=12301) 2024-11-11T07:31:38,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. as already flushing 2024-11-11T07:31:38,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:38,746 INFO [Thread-2286 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2286. 2024-11-11T07:31:38,746 DEBUG [Thread-2286 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:38,746 DEBUG [Thread-2286 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,746 DEBUG [Thread-2286 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:38,746 INFO [Thread-2284 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2284. 2024-11-11T07:31:38,746 DEBUG [Thread-2284 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:38,746 DEBUG [Thread-2284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,746 DEBUG [Thread-2284 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,747 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:38,749 INFO [Thread-2269 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2269. 2024-11-11T07:31:38,749 DEBUG [Thread-2269 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:38,749 DEBUG [Thread-2269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,749 DEBUG [Thread-2269 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,750 INFO [Thread-2271 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2271. 2024-11-11T07:31:38,750 DEBUG [Thread-2271 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:38,750 DEBUG [Thread-2271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,750 DEBUG [Thread-2271 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:38,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:38,752 INFO [Thread-2273 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2273. 2024-11-11T07:31:38,752 DEBUG [Thread-2273 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:38,752 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,752 DEBUG [Thread-2273 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,752 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:38,753 INFO [Thread-2280 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2280. 2024-11-11T07:31:38,753 DEBUG [Thread-2280 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:38,753 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,753 DEBUG [Thread-2280 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,754 INFO [Thread-2267 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2267. 2024-11-11T07:31:38,754 DEBUG [Thread-2267 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:38,754 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,754 DEBUG [Thread-2267 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:38,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:38,755 INFO [Thread-2275 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2275. 2024-11-11T07:31:38,755 DEBUG [Thread-2275 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:38,755 DEBUG [Thread-2275 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,755 DEBUG [Thread-2275 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,755 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:38,760 INFO [Thread-2282 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2282. 2024-11-11T07:31:38,760 DEBUG [Thread-2282 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:38,760 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,761 DEBUG [Thread-2282 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,761 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:38,765 INFO [Thread-2278 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2278. 2024-11-11T07:31:38,765 DEBUG [Thread-2278 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:31:38,766 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,766 DEBUG [Thread-2278 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:38,766 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:39,103 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/42baff9a8a7b40068fcc1ea12ea886aa 2024-11-11T07:31:39,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/2bea904caaa54f8cb0e5aecc3cab9097 is 50, key is test_row_0/C:col10/1731310297551/Put/seqid=0 2024-11-11T07:31:39,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742468_1644 (size=12301) 2024-11-11T07:31:39,512 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/2bea904caaa54f8cb0e5aecc3cab9097 2024-11-11T07:31:39,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/9cf630e8a7f2424986de3e1b6ed7d91e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/9cf630e8a7f2424986de3e1b6ed7d91e 2024-11-11T07:31:39,518 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/9cf630e8a7f2424986de3e1b6ed7d91e, entries=150, sequenceid=629, filesize=12.0 K 2024-11-11T07:31:39,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/42baff9a8a7b40068fcc1ea12ea886aa as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/42baff9a8a7b40068fcc1ea12ea886aa 2024-11-11T07:31:39,523 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/42baff9a8a7b40068fcc1ea12ea886aa, entries=150, sequenceid=629, filesize=12.0 K 2024-11-11T07:31:39,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/2bea904caaa54f8cb0e5aecc3cab9097 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/2bea904caaa54f8cb0e5aecc3cab9097 2024-11-11T07:31:39,526 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/2bea904caaa54f8cb0e5aecc3cab9097, entries=150, sequenceid=629, filesize=12.0 K 2024-11-11T07:31:39,527 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 752742eb661c603f835a96396e7673f2 in 945ms, sequenceid=629, compaction requested=true 2024-11-11T07:31:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.HRegion(2603): Flush status journal for 752742eb661c603f835a96396e7673f2: 2024-11-11T07:31:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:39,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-11T07:31:39,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=146 2024-11-11T07:31:39,529 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-11T07:31:39,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0380 sec 2024-11-11T07:31:39,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=145, table=TestAcidGuarantees in 2.0450 sec 2024-11-11T07:31:39,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-11T07:31:39,645 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 96 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 99 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 100 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 98 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 114 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 776 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2328 rows 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 786 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2358 rows 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 785 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2334 rows 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 778 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2333 rows 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 779 2024-11-11T07:31:39,645 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2328 rows 2024-11-11T07:31:39,646 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T07:31:39,646 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:31:39,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:39,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:39,646 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:31:39,646 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T07:31:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:39,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-11T07:31:39,649 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310299649"}]},"ts":"1731310299649"} 2024-11-11T07:31:39,650 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T07:31:39,651 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T07:31:39,651 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:31:39,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=752742eb661c603f835a96396e7673f2, UNASSIGN}] 2024-11-11T07:31:39,653 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=752742eb661c603f835a96396e7673f2, UNASSIGN 2024-11-11T07:31:39,653 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=752742eb661c603f835a96396e7673f2, regionState=CLOSING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:39,655 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=752742eb661c603f835a96396e7673f2, UNASSIGN because future has completed 2024-11-11T07:31:39,655 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:31:39,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:31:39,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-11T07:31:39,807 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:39,807 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:31:39,807 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 752742eb661c603f835a96396e7673f2, disabling compactions & flushes 2024-11-11T07:31:39,807 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:39,807 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:39,807 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. after waiting 0 ms 2024-11-11T07:31:39,807 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:39,807 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(2902): Flushing 752742eb661c603f835a96396e7673f2 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T07:31:39,807 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=A 2024-11-11T07:31:39,807 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:39,807 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=B 2024-11-11T07:31:39,807 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:39,807 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 752742eb661c603f835a96396e7673f2, store=C 2024-11-11T07:31:39,807 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:39,811 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/a6e746ae187d4d049f3c76149c5c7b4e is 50, key is test_row_0/A:col10/1731310298747/Put/seqid=0 2024-11-11T07:31:39,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742469_1645 (size=12301) 2024-11-11T07:31:39,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-11T07:31:40,214 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/a6e746ae187d4d049f3c76149c5c7b4e 2024-11-11T07:31:40,220 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/45aace9df1a1454ab504925c66d12851 is 50, key is test_row_0/B:col10/1731310298747/Put/seqid=0 2024-11-11T07:31:40,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742470_1646 (size=12301) 2024-11-11T07:31:40,228 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/45aace9df1a1454ab504925c66d12851 2024-11-11T07:31:40,234 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/ab86aa65310845fb9676db92a123f9d7 is 50, key is test_row_0/C:col10/1731310298747/Put/seqid=0 2024-11-11T07:31:40,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742471_1647 (size=12301) 2024-11-11T07:31:40,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-11T07:31:40,638 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=642 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/ab86aa65310845fb9676db92a123f9d7 2024-11-11T07:31:40,641 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/A/a6e746ae187d4d049f3c76149c5c7b4e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/a6e746ae187d4d049f3c76149c5c7b4e 2024-11-11T07:31:40,644 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/a6e746ae187d4d049f3c76149c5c7b4e, entries=150, sequenceid=642, filesize=12.0 K 2024-11-11T07:31:40,644 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/B/45aace9df1a1454ab504925c66d12851 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/45aace9df1a1454ab504925c66d12851 2024-11-11T07:31:40,647 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/45aace9df1a1454ab504925c66d12851, entries=150, sequenceid=642, filesize=12.0 K 2024-11-11T07:31:40,647 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/.tmp/C/ab86aa65310845fb9676db92a123f9d7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ab86aa65310845fb9676db92a123f9d7 2024-11-11T07:31:40,650 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ab86aa65310845fb9676db92a123f9d7, entries=150, sequenceid=642, filesize=12.0 K 2024-11-11T07:31:40,651 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 752742eb661c603f835a96396e7673f2 in 844ms, sequenceid=642, compaction requested=true 2024-11-11T07:31:40,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/a8c74bd1523a47a486f4be4a2b574692, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c3f5ebb9c8894ea7a474ef5e28097c0a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1c25d5e8aab34d559fbec171823f349a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/82338252e26a4540aa166cdad1ec6cc7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/508a6fde64754edd85203947da483c75, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/802c3310b71541198a6e4f2e5d0a3bca, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/705365d123da473687dd5090ccf835e0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/cf4555952ea646cb9b1b200d14f9bba5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/09fc52699fd74ab58997da2ad717cb76, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/7094ae39a54a40069c1412ea19f9a822, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/b73446016fd44eb8913a0802f081fd76, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1ebec904021646999a1a466d04806633, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/cdca3cb0362f4328ba0773d17067a391, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/5cb3079429a34deca3317f13f5113930, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/dacaa6d953484675ab7797f2792555a0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/d8f3347ddb584684a50d11c2dc4b593d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1503bad9045f4b5db163acfd1daf080a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/5163ed8867c347e99d1384e84e3ffd73, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/a2017fa3eac747fd916955459d871bba, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/feb2df4ceff9471ea7146d38d918b035, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/d8d7f7625858481b9dbc61c535307092, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/8e94d36673c04896a4f7acbd8f4be344, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/bf4cda33e673490dac8fd4df1fc724b6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/3b78f7e1ea8449b39ce3e8bdfaec75b0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/12a5617092e04bcf9dcabe618594c776, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/654d3041dc21490ba95c5793cf87e476, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/aa357d0db7f8462587b9d2bdc443d22e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/bcf4a79854dc4cdbab07d1c9998aacdd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/90b9e08da1e8403a88c8930cbf8ac048, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/5528484381824ccf8af013bd16135264, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/4ba7e3c7735a4cdba9ef0b9a78f90ed8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1df37a9980b44873aa72078cca2107bf, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c7fa4f5edaed4b8298bb8e23c7f6e506, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1febd04466b34b2eaa7750928e099170, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c5d79816126c4016b76f24138393d845, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c0731e8603e3404cb2c6281ba48862c3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/bead7466e5b84629b7a1a8b2a8a9378b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/64e47e2c19ec46d8ada549e5e0fc2f80, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/046e6771cb68442cbeda176316c0ab29, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/501abd5815ad41deac140fb01d63e936, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/f34b18e72b01479698b4a26a45ff8d55] to archive 2024-11-11T07:31:40,653 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:31:40,654 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/a8c74bd1523a47a486f4be4a2b574692 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/a8c74bd1523a47a486f4be4a2b574692 2024-11-11T07:31:40,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c3f5ebb9c8894ea7a474ef5e28097c0a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c3f5ebb9c8894ea7a474ef5e28097c0a 2024-11-11T07:31:40,656 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1c25d5e8aab34d559fbec171823f349a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1c25d5e8aab34d559fbec171823f349a 2024-11-11T07:31:40,657 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/82338252e26a4540aa166cdad1ec6cc7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/82338252e26a4540aa166cdad1ec6cc7 2024-11-11T07:31:40,658 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/508a6fde64754edd85203947da483c75 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/508a6fde64754edd85203947da483c75 2024-11-11T07:31:40,659 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/802c3310b71541198a6e4f2e5d0a3bca to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/802c3310b71541198a6e4f2e5d0a3bca 2024-11-11T07:31:40,660 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/705365d123da473687dd5090ccf835e0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/705365d123da473687dd5090ccf835e0 2024-11-11T07:31:40,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/cf4555952ea646cb9b1b200d14f9bba5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/cf4555952ea646cb9b1b200d14f9bba5 2024-11-11T07:31:40,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/09fc52699fd74ab58997da2ad717cb76 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/09fc52699fd74ab58997da2ad717cb76 2024-11-11T07:31:40,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/7094ae39a54a40069c1412ea19f9a822 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/7094ae39a54a40069c1412ea19f9a822 2024-11-11T07:31:40,663 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/b73446016fd44eb8913a0802f081fd76 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/b73446016fd44eb8913a0802f081fd76 2024-11-11T07:31:40,664 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1ebec904021646999a1a466d04806633 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1ebec904021646999a1a466d04806633 2024-11-11T07:31:40,665 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/cdca3cb0362f4328ba0773d17067a391 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/cdca3cb0362f4328ba0773d17067a391 2024-11-11T07:31:40,666 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/5cb3079429a34deca3317f13f5113930 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/5cb3079429a34deca3317f13f5113930 2024-11-11T07:31:40,667 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/dacaa6d953484675ab7797f2792555a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/dacaa6d953484675ab7797f2792555a0 2024-11-11T07:31:40,668 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/d8f3347ddb584684a50d11c2dc4b593d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/d8f3347ddb584684a50d11c2dc4b593d 2024-11-11T07:31:40,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1503bad9045f4b5db163acfd1daf080a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1503bad9045f4b5db163acfd1daf080a 2024-11-11T07:31:40,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/5163ed8867c347e99d1384e84e3ffd73 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/5163ed8867c347e99d1384e84e3ffd73 2024-11-11T07:31:40,670 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/a2017fa3eac747fd916955459d871bba to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/a2017fa3eac747fd916955459d871bba 2024-11-11T07:31:40,671 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/feb2df4ceff9471ea7146d38d918b035 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/feb2df4ceff9471ea7146d38d918b035 2024-11-11T07:31:40,672 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/d8d7f7625858481b9dbc61c535307092 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/d8d7f7625858481b9dbc61c535307092 2024-11-11T07:31:40,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/8e94d36673c04896a4f7acbd8f4be344 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/8e94d36673c04896a4f7acbd8f4be344 2024-11-11T07:31:40,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/bf4cda33e673490dac8fd4df1fc724b6 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/bf4cda33e673490dac8fd4df1fc724b6 2024-11-11T07:31:40,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/3b78f7e1ea8449b39ce3e8bdfaec75b0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/3b78f7e1ea8449b39ce3e8bdfaec75b0 2024-11-11T07:31:40,675 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/12a5617092e04bcf9dcabe618594c776 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/12a5617092e04bcf9dcabe618594c776 2024-11-11T07:31:40,676 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/654d3041dc21490ba95c5793cf87e476 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/654d3041dc21490ba95c5793cf87e476 2024-11-11T07:31:40,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/aa357d0db7f8462587b9d2bdc443d22e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/aa357d0db7f8462587b9d2bdc443d22e 2024-11-11T07:31:40,678 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/bcf4a79854dc4cdbab07d1c9998aacdd to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/bcf4a79854dc4cdbab07d1c9998aacdd 2024-11-11T07:31:40,679 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/90b9e08da1e8403a88c8930cbf8ac048 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/90b9e08da1e8403a88c8930cbf8ac048 2024-11-11T07:31:40,680 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/5528484381824ccf8af013bd16135264 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/5528484381824ccf8af013bd16135264 2024-11-11T07:31:40,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/4ba7e3c7735a4cdba9ef0b9a78f90ed8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/4ba7e3c7735a4cdba9ef0b9a78f90ed8 2024-11-11T07:31:40,681 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1df37a9980b44873aa72078cca2107bf to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1df37a9980b44873aa72078cca2107bf 2024-11-11T07:31:40,682 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c7fa4f5edaed4b8298bb8e23c7f6e506 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c7fa4f5edaed4b8298bb8e23c7f6e506 2024-11-11T07:31:40,684 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1febd04466b34b2eaa7750928e099170 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/1febd04466b34b2eaa7750928e099170 2024-11-11T07:31:40,685 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c5d79816126c4016b76f24138393d845 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c5d79816126c4016b76f24138393d845 2024-11-11T07:31:40,686 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c0731e8603e3404cb2c6281ba48862c3 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/c0731e8603e3404cb2c6281ba48862c3 2024-11-11T07:31:40,687 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/bead7466e5b84629b7a1a8b2a8a9378b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/bead7466e5b84629b7a1a8b2a8a9378b 2024-11-11T07:31:40,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/64e47e2c19ec46d8ada549e5e0fc2f80 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/64e47e2c19ec46d8ada549e5e0fc2f80 2024-11-11T07:31:40,688 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/046e6771cb68442cbeda176316c0ab29 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/046e6771cb68442cbeda176316c0ab29 2024-11-11T07:31:40,689 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/501abd5815ad41deac140fb01d63e936 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/501abd5815ad41deac140fb01d63e936 2024-11-11T07:31:40,690 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/f34b18e72b01479698b4a26a45ff8d55 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/f34b18e72b01479698b4a26a45ff8d55 2024-11-11T07:31:40,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/eb15f13dc16e4a65ace88887f5e24aac, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/da7432f9a07d48be99bcc55e77373834, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f91c27309c744c4cb47043392c3454bf, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/fe9c1db1950f428fb237c9889253c5c7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f19d578d50ee4c8183b77f801e5a497e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/a097971806514ce7b7af81cd7db44668, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/08a4d91eb78d48fca987abd890f0ba84, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/d259c81ec04647cd861d95aed626a3fb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0fae72c8f7c9455d86833c3a2a7dfc2c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ce2fff3e5a044496bbc0fb31a705c963, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/e869c048d5064130bbd53492b15f2e19, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/16a94cfc094a479285e5c47f632bb347, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/e45c3b872384449f8d1063ce809c4f7e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0254e6afb50d481ea4909a9840a655b4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/515ba2b710d8492592e1a600c325d47f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/72255fd6f6b04e5e904ce55655f0d539, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/100de80240014a439bf2121778a70550, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/386c48fc723e48b0b189af6d4244235a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f64752d3383d49278251bd2825da54cb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/c4389f0cbd354c3d8a7439e8b2658455, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ac0e0a0f286f43c3980489b3a8adab3e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/450a5d14f617463cac8a815b121c0198, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/924b8a2fb34e46069272a526909d7eca, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/fc4353e762114f9ca1eabec3de4a2ff1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/33385d698e014bf9af078140eb84fd66, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/209557f5541e4330999cda1b45f33942, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/7724d25f256449aa9c98fa216851cde9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/4011a264343143bdbfe3dd161e24207b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/74ac5d4531f84ce4a746aa317d91703b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/2a0ab674bc5d4e46b8a2c0d50db41995, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ea439d3883d34d1081933f8b52171b69, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/7e85068ded3045808a8ff9bc92094c73, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/91ca48bbe5d744db8edebff9ab555d0b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/dc935753a39344e8a3c7c7d314504578, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/458a5094678a4f72b6da3463fc252110, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/bc1f84dd2f224d6a87df889efa9d38a7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/795b39e705d94160981d6dadc41c6cf7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0a6368cb880c4b58a464895555759865, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/153d8fd221ec4896805a5fbba50a1d35, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/57d1ce4efc694a60badf99a588a6b5a5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/4c8c6244893240cc98c75efdb622a832] to archive 2024-11-11T07:31:40,693 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:31:40,694 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/eb15f13dc16e4a65ace88887f5e24aac to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/eb15f13dc16e4a65ace88887f5e24aac 2024-11-11T07:31:40,695 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/da7432f9a07d48be99bcc55e77373834 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/da7432f9a07d48be99bcc55e77373834 2024-11-11T07:31:40,696 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f91c27309c744c4cb47043392c3454bf to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f91c27309c744c4cb47043392c3454bf 2024-11-11T07:31:40,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/fe9c1db1950f428fb237c9889253c5c7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/fe9c1db1950f428fb237c9889253c5c7 2024-11-11T07:31:40,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f19d578d50ee4c8183b77f801e5a497e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f19d578d50ee4c8183b77f801e5a497e 2024-11-11T07:31:40,698 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/a097971806514ce7b7af81cd7db44668 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/a097971806514ce7b7af81cd7db44668 2024-11-11T07:31:40,699 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/08a4d91eb78d48fca987abd890f0ba84 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/08a4d91eb78d48fca987abd890f0ba84 2024-11-11T07:31:40,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/d259c81ec04647cd861d95aed626a3fb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/d259c81ec04647cd861d95aed626a3fb 2024-11-11T07:31:40,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0fae72c8f7c9455d86833c3a2a7dfc2c to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0fae72c8f7c9455d86833c3a2a7dfc2c 2024-11-11T07:31:40,703 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ce2fff3e5a044496bbc0fb31a705c963 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ce2fff3e5a044496bbc0fb31a705c963 2024-11-11T07:31:40,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/e869c048d5064130bbd53492b15f2e19 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/e869c048d5064130bbd53492b15f2e19 2024-11-11T07:31:40,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/16a94cfc094a479285e5c47f632bb347 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/16a94cfc094a479285e5c47f632bb347 2024-11-11T07:31:40,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/e45c3b872384449f8d1063ce809c4f7e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/e45c3b872384449f8d1063ce809c4f7e 2024-11-11T07:31:40,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0254e6afb50d481ea4909a9840a655b4 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0254e6afb50d481ea4909a9840a655b4 2024-11-11T07:31:40,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/515ba2b710d8492592e1a600c325d47f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/515ba2b710d8492592e1a600c325d47f 2024-11-11T07:31:40,708 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/72255fd6f6b04e5e904ce55655f0d539 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/72255fd6f6b04e5e904ce55655f0d539 2024-11-11T07:31:40,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/100de80240014a439bf2121778a70550 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/100de80240014a439bf2121778a70550 2024-11-11T07:31:40,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/386c48fc723e48b0b189af6d4244235a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/386c48fc723e48b0b189af6d4244235a 2024-11-11T07:31:40,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f64752d3383d49278251bd2825da54cb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/f64752d3383d49278251bd2825da54cb 2024-11-11T07:31:40,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/c4389f0cbd354c3d8a7439e8b2658455 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/c4389f0cbd354c3d8a7439e8b2658455 2024-11-11T07:31:40,713 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ac0e0a0f286f43c3980489b3a8adab3e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ac0e0a0f286f43c3980489b3a8adab3e 2024-11-11T07:31:40,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/450a5d14f617463cac8a815b121c0198 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/450a5d14f617463cac8a815b121c0198 2024-11-11T07:31:40,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/924b8a2fb34e46069272a526909d7eca to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/924b8a2fb34e46069272a526909d7eca 2024-11-11T07:31:40,715 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/fc4353e762114f9ca1eabec3de4a2ff1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/fc4353e762114f9ca1eabec3de4a2ff1 2024-11-11T07:31:40,716 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/33385d698e014bf9af078140eb84fd66 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/33385d698e014bf9af078140eb84fd66 2024-11-11T07:31:40,717 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/209557f5541e4330999cda1b45f33942 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/209557f5541e4330999cda1b45f33942 2024-11-11T07:31:40,718 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/7724d25f256449aa9c98fa216851cde9 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/7724d25f256449aa9c98fa216851cde9 2024-11-11T07:31:40,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/4011a264343143bdbfe3dd161e24207b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/4011a264343143bdbfe3dd161e24207b 2024-11-11T07:31:40,720 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/74ac5d4531f84ce4a746aa317d91703b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/74ac5d4531f84ce4a746aa317d91703b 2024-11-11T07:31:40,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/2a0ab674bc5d4e46b8a2c0d50db41995 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/2a0ab674bc5d4e46b8a2c0d50db41995 2024-11-11T07:31:40,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ea439d3883d34d1081933f8b52171b69 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/ea439d3883d34d1081933f8b52171b69 2024-11-11T07:31:40,722 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/7e85068ded3045808a8ff9bc92094c73 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/7e85068ded3045808a8ff9bc92094c73 2024-11-11T07:31:40,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/91ca48bbe5d744db8edebff9ab555d0b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/91ca48bbe5d744db8edebff9ab555d0b 2024-11-11T07:31:40,724 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/dc935753a39344e8a3c7c7d314504578 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/dc935753a39344e8a3c7c7d314504578 2024-11-11T07:31:40,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/458a5094678a4f72b6da3463fc252110 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/458a5094678a4f72b6da3463fc252110 2024-11-11T07:31:40,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/bc1f84dd2f224d6a87df889efa9d38a7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/bc1f84dd2f224d6a87df889efa9d38a7 2024-11-11T07:31:40,726 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/795b39e705d94160981d6dadc41c6cf7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/795b39e705d94160981d6dadc41c6cf7 2024-11-11T07:31:40,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0a6368cb880c4b58a464895555759865 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/0a6368cb880c4b58a464895555759865 2024-11-11T07:31:40,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/153d8fd221ec4896805a5fbba50a1d35 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/153d8fd221ec4896805a5fbba50a1d35 2024-11-11T07:31:40,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/57d1ce4efc694a60badf99a588a6b5a5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/57d1ce4efc694a60badf99a588a6b5a5 2024-11-11T07:31:40,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/4c8c6244893240cc98c75efdb622a832 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/4c8c6244893240cc98c75efdb622a832 2024-11-11T07:31:40,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/20bd7bf2e7f143fc817e9ba7b81ebd4d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/16c334de86b546dd87dee814c6150891, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ad52b78c19854fada904a2d1d7a6a3c8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/aeb1f3ab761446ba9994a6a6e7cf8af3, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/c4f3dc29fb00470f9b3ce48450245947, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/933c77e550514821984a598e08848d77, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/c5090ac8f5904427a931339e414e6233, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/41e39d0e69e2463c907090ecfb8464a7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/526c99f07def409e866a63b093623eac, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8977f86c10b24ab0b7993d0ba1548e59, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/5ed89a432ea34e8098920b481b7be052, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/70ade268a79a44dab46512cdbb3ccc56, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ba1b173af7d94d6ebf1f395f939a4161, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/587dc6c96cb6401999fe17dfc0fc5e7c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/faf1bdc5c1a54e5e8837745dcabfebeb, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ef4a679fdb534b1093386ecda30485ac, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/06e5f3bc4e6a49518551f9d31e926301, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/599b985472af40b68b4def1ada1d5c30, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/77eda05fb40f4f23b7a4394fadc45400, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/de419c5566ea4085b15fdb74cecd6946, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/6615c9d8947f48b7ae8d1fe36c248158, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8c0bbd74b7a647e1b5a6b67d03390cf4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a9ce33ca96ba43fdb9dee59d214b8b97, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/666f4b4bbbef43ec9901eb5c106454af, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a9c85ac6d9114b7587d116c1633bb573, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/21a9969ee05748388005338bd6ee1dd5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/195db8975fda4b44a118f5d4229c55a5, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/96a8ffc9a153448d9642ea7670ae144d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/d88c075faa7a4df595eeabcb92ea6581, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ec3392ee52f64f0684d8ffbe88c494a8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/4cd066ea33a744f2b73dabca12b0dc9f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/73fcc1ef5ff64231b614e0d568ae20d8, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/49cb389fb2964e6d9ab5e0fa62d6d6d2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8f1ee694842d45b09ea814fa2027aa4f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/e89fda178ef54e0096f570c1f82a2d11, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/fff7b4e04b02431d888a09f54cb7a27b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a2f91c438f524db7a34948669dab1704, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/9bad78bbae5242938a770e5f4bea22a4, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/2215253d59f64b08b087785ee815490d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/9765381606af49bfbb851caec09af9d9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/5b703fc4efda481695e7d45ac6f8e81f] to archive 2024-11-11T07:31:40,732 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:31:40,733 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/20bd7bf2e7f143fc817e9ba7b81ebd4d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/20bd7bf2e7f143fc817e9ba7b81ebd4d 2024-11-11T07:31:40,734 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/16c334de86b546dd87dee814c6150891 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/16c334de86b546dd87dee814c6150891 2024-11-11T07:31:40,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ad52b78c19854fada904a2d1d7a6a3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ad52b78c19854fada904a2d1d7a6a3c8 2024-11-11T07:31:40,736 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/aeb1f3ab761446ba9994a6a6e7cf8af3 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/aeb1f3ab761446ba9994a6a6e7cf8af3 2024-11-11T07:31:40,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/c4f3dc29fb00470f9b3ce48450245947 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/c4f3dc29fb00470f9b3ce48450245947 2024-11-11T07:31:40,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/933c77e550514821984a598e08848d77 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/933c77e550514821984a598e08848d77 2024-11-11T07:31:40,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/c5090ac8f5904427a931339e414e6233 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/c5090ac8f5904427a931339e414e6233 2024-11-11T07:31:40,739 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/41e39d0e69e2463c907090ecfb8464a7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/41e39d0e69e2463c907090ecfb8464a7 2024-11-11T07:31:40,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/526c99f07def409e866a63b093623eac to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/526c99f07def409e866a63b093623eac 2024-11-11T07:31:40,741 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8977f86c10b24ab0b7993d0ba1548e59 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8977f86c10b24ab0b7993d0ba1548e59 2024-11-11T07:31:40,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/5ed89a432ea34e8098920b481b7be052 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/5ed89a432ea34e8098920b481b7be052 2024-11-11T07:31:40,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/70ade268a79a44dab46512cdbb3ccc56 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/70ade268a79a44dab46512cdbb3ccc56 2024-11-11T07:31:40,743 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ba1b173af7d94d6ebf1f395f939a4161 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ba1b173af7d94d6ebf1f395f939a4161 2024-11-11T07:31:40,744 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/587dc6c96cb6401999fe17dfc0fc5e7c to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/587dc6c96cb6401999fe17dfc0fc5e7c 2024-11-11T07:31:40,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/faf1bdc5c1a54e5e8837745dcabfebeb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/faf1bdc5c1a54e5e8837745dcabfebeb 2024-11-11T07:31:40,745 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ef4a679fdb534b1093386ecda30485ac to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ef4a679fdb534b1093386ecda30485ac 2024-11-11T07:31:40,746 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/06e5f3bc4e6a49518551f9d31e926301 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/06e5f3bc4e6a49518551f9d31e926301 2024-11-11T07:31:40,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/599b985472af40b68b4def1ada1d5c30 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/599b985472af40b68b4def1ada1d5c30 2024-11-11T07:31:40,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/77eda05fb40f4f23b7a4394fadc45400 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/77eda05fb40f4f23b7a4394fadc45400 2024-11-11T07:31:40,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/de419c5566ea4085b15fdb74cecd6946 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/de419c5566ea4085b15fdb74cecd6946 2024-11-11T07:31:40,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/6615c9d8947f48b7ae8d1fe36c248158 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/6615c9d8947f48b7ae8d1fe36c248158 2024-11-11T07:31:40,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8c0bbd74b7a647e1b5a6b67d03390cf4 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8c0bbd74b7a647e1b5a6b67d03390cf4 2024-11-11T07:31:40,751 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a9ce33ca96ba43fdb9dee59d214b8b97 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a9ce33ca96ba43fdb9dee59d214b8b97 2024-11-11T07:31:40,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/666f4b4bbbef43ec9901eb5c106454af to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/666f4b4bbbef43ec9901eb5c106454af 2024-11-11T07:31:40,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a9c85ac6d9114b7587d116c1633bb573 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a9c85ac6d9114b7587d116c1633bb573 2024-11-11T07:31:40,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/21a9969ee05748388005338bd6ee1dd5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/21a9969ee05748388005338bd6ee1dd5 2024-11-11T07:31:40,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/195db8975fda4b44a118f5d4229c55a5 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/195db8975fda4b44a118f5d4229c55a5 2024-11-11T07:31:40,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/96a8ffc9a153448d9642ea7670ae144d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/96a8ffc9a153448d9642ea7670ae144d 2024-11-11T07:31:40,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/d88c075faa7a4df595eeabcb92ea6581 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/d88c075faa7a4df595eeabcb92ea6581 2024-11-11T07:31:40,756 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ec3392ee52f64f0684d8ffbe88c494a8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ec3392ee52f64f0684d8ffbe88c494a8 2024-11-11T07:31:40,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/4cd066ea33a744f2b73dabca12b0dc9f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/4cd066ea33a744f2b73dabca12b0dc9f 2024-11-11T07:31:40,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/73fcc1ef5ff64231b614e0d568ae20d8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/73fcc1ef5ff64231b614e0d568ae20d8 2024-11-11T07:31:40,759 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/49cb389fb2964e6d9ab5e0fa62d6d6d2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/49cb389fb2964e6d9ab5e0fa62d6d6d2 2024-11-11T07:31:40,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8f1ee694842d45b09ea814fa2027aa4f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/8f1ee694842d45b09ea814fa2027aa4f 2024-11-11T07:31:40,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/e89fda178ef54e0096f570c1f82a2d11 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/e89fda178ef54e0096f570c1f82a2d11 2024-11-11T07:31:40,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/fff7b4e04b02431d888a09f54cb7a27b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/fff7b4e04b02431d888a09f54cb7a27b 2024-11-11T07:31:40,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a2f91c438f524db7a34948669dab1704 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/a2f91c438f524db7a34948669dab1704 2024-11-11T07:31:40,763 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/9bad78bbae5242938a770e5f4bea22a4 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/9bad78bbae5242938a770e5f4bea22a4 2024-11-11T07:31:40,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/2215253d59f64b08b087785ee815490d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/2215253d59f64b08b087785ee815490d 2024-11-11T07:31:40,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/9765381606af49bfbb851caec09af9d9 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/9765381606af49bfbb851caec09af9d9 2024-11-11T07:31:40,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/5b703fc4efda481695e7d45ac6f8e81f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/5b703fc4efda481695e7d45ac6f8e81f 2024-11-11T07:31:40,773 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/recovered.edits/645.seqid, newMaxSeqId=645, maxSeqId=1 2024-11-11T07:31:40,774 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. 2024-11-11T07:31:40,774 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 752742eb661c603f835a96396e7673f2: Waiting for close lock at 1731310299807Running coprocessor pre-close hooks at 1731310299807Disabling compacts and flushes for region at 1731310299807Disabling writes for close at 1731310299807Obtaining lock to block concurrent updates at 1731310299807Preparing flush snapshotting stores in 752742eb661c603f835a96396e7673f2 at 1731310299807Finished memstore snapshotting TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2., syncing WAL and waiting on mvcc, flushsize=dataSize=68700, getHeapSize=204720, getOffHeapSize=0, getCellsCount=1500 at 1731310299807Flushing stores of TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2. at 1731310299808 (+1 ms)Flushing 752742eb661c603f835a96396e7673f2/A: creating writer at 1731310299808Flushing 752742eb661c603f835a96396e7673f2/A: appending metadata at 1731310299810 (+2 ms)Flushing 752742eb661c603f835a96396e7673f2/A: closing flushed file at 1731310299810Flushing 752742eb661c603f835a96396e7673f2/B: creating writer at 1731310300217 (+407 ms)Flushing 752742eb661c603f835a96396e7673f2/B: appending metadata at 1731310300220 (+3 ms)Flushing 752742eb661c603f835a96396e7673f2/B: closing flushed file at 1731310300220Flushing 752742eb661c603f835a96396e7673f2/C: creating writer at 1731310300231 (+11 ms)Flushing 752742eb661c603f835a96396e7673f2/C: appending metadata at 1731310300233 (+2 ms)Flushing 752742eb661c603f835a96396e7673f2/C: closing flushed file at 1731310300233Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a3fedd0: reopening flushed file at 1731310300640 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@78bcd095: reopening flushed file at 1731310300644 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2ba46508: reopening flushed file at 1731310300647 (+3 ms)Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=0 B/0 for 752742eb661c603f835a96396e7673f2 in 844ms, sequenceid=642, compaction requested=true at 1731310300651 (+4 ms)Writing region close event to WAL at 1731310300770 (+119 ms)Running coprocessor post-close hooks at 1731310300774 (+4 ms)Closed at 1731310300774 2024-11-11T07:31:40,776 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 752742eb661c603f835a96396e7673f2 2024-11-11T07:31:40,776 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=752742eb661c603f835a96396e7673f2, regionState=CLOSED 2024-11-11T07:31:40,778 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:31:40,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-11T07:31:40,780 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 752742eb661c603f835a96396e7673f2, server=ce2cf0244921,33345,1731310155369 in 1.1240 sec 2024-11-11T07:31:40,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-11T07:31:40,782 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=752742eb661c603f835a96396e7673f2, UNASSIGN in 1.1280 sec 2024-11-11T07:31:40,783 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-11T07:31:40,784 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1310 sec 2024-11-11T07:31:40,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-11T07:31:40,785 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310300785"}]},"ts":"1731310300785"} 2024-11-11T07:31:40,787 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T07:31:40,787 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T07:31:40,791 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1420 sec 2024-11-11T07:31:41,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-11T07:31:41,795 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:41,796 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T07:31:41,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:41,798 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=151, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-11T07:31:41,798 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=151, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:41,801 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2 2024-11-11T07:31:41,802 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/recovered.edits] 2024-11-11T07:31:41,804 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/45d8b1383dcf437f87af84bd4137b3ef to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/45d8b1383dcf437f87af84bd4137b3ef 2024-11-11T07:31:41,805 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/91553cf554674f278552b49f2cf8a334 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/91553cf554674f278552b49f2cf8a334 2024-11-11T07:31:41,806 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/9cf630e8a7f2424986de3e1b6ed7d91e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/9cf630e8a7f2424986de3e1b6ed7d91e 2024-11-11T07:31:41,806 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/a6e746ae187d4d049f3c76149c5c7b4e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/A/a6e746ae187d4d049f3c76149c5c7b4e 2024-11-11T07:31:41,808 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/42baff9a8a7b40068fcc1ea12ea886aa to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/42baff9a8a7b40068fcc1ea12ea886aa 2024-11-11T07:31:41,809 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/45aace9df1a1454ab504925c66d12851 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/45aace9df1a1454ab504925c66d12851 2024-11-11T07:31:41,810 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/5e4c415e93334060979879ea6c63094a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/5e4c415e93334060979879ea6c63094a 2024-11-11T07:31:41,810 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/d5367e3f5511460bac5e8f2d42865a88 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/B/d5367e3f5511460bac5e8f2d42865a88 2024-11-11T07:31:41,812 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/2bea904caaa54f8cb0e5aecc3cab9097 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/2bea904caaa54f8cb0e5aecc3cab9097 2024-11-11T07:31:41,813 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/589b35f769b8437fac9bcdfb620d4dfb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/589b35f769b8437fac9bcdfb620d4dfb 2024-11-11T07:31:41,813 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ab86aa65310845fb9676db92a123f9d7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/C/ab86aa65310845fb9676db92a123f9d7 2024-11-11T07:31:41,815 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/recovered.edits/645.seqid to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2/recovered.edits/645.seqid 2024-11-11T07:31:41,816 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/752742eb661c603f835a96396e7673f2 2024-11-11T07:31:41,816 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T07:31:41,818 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=151, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:41,820 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T07:31:41,822 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T07:31:41,823 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=151, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:41,823 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T07:31:41,823 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731310301823"}]},"ts":"9223372036854775807"} 2024-11-11T07:31:41,825 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T07:31:41,825 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 752742eb661c603f835a96396e7673f2, NAME => 'TestAcidGuarantees,,1731310276476.752742eb661c603f835a96396e7673f2.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T07:31:41,825 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T07:31:41,825 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731310301825"}]},"ts":"9223372036854775807"} 2024-11-11T07:31:41,827 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T07:31:41,828 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=151, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:41,829 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 32 msec 2024-11-11T07:31:41,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-11-11T07:31:41,905 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T07:31:41,908 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:41,916 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=250 (was 247) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=872 (was 884), ProcessCount=11 (was 11), AvailableMemoryMB=2409 (was 2435) 2024-11-11T07:31:41,926 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=250, OpenFileDescriptor=447, MaxFileDescriptor=1048576, SystemLoadAverage=872, ProcessCount=11, AvailableMemoryMB=2409 2024-11-11T07:31:41,927 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:31:41,928 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:31:41,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:41,929 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T07:31:41,929 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:41,929 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 152 2024-11-11T07:31:41,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T07:31:41,930 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T07:31:41,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742472_1648 (size=1061) 2024-11-11T07:31:42,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T07:31:42,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T07:31:42,337 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e 2024-11-11T07:31:42,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742473_1649 (size=53) 2024-11-11T07:31:42,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:31:42,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 298c53e334bb69b7b36af68612f3d3c8, disabling compactions & flushes 2024-11-11T07:31:42,342 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:42,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:42,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. after waiting 0 ms 2024-11-11T07:31:42,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:42,342 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:42,342 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 298c53e334bb69b7b36af68612f3d3c8: Waiting for close lock at 1731310302342Disabling compacts and flushes for region at 1731310302342Disabling writes for close at 1731310302342Writing region close event to WAL at 1731310302342Closed at 1731310302342 2024-11-11T07:31:42,343 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T07:31:42,343 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731310302343"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731310302343"}]},"ts":"1731310302343"} 2024-11-11T07:31:42,345 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-11T07:31:42,346 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T07:31:42,346 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310302346"}]},"ts":"1731310302346"} 2024-11-11T07:31:42,347 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-11T07:31:42,347 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=298c53e334bb69b7b36af68612f3d3c8, ASSIGN}] 2024-11-11T07:31:42,349 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=298c53e334bb69b7b36af68612f3d3c8, ASSIGN 2024-11-11T07:31:42,350 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=298c53e334bb69b7b36af68612f3d3c8, ASSIGN; state=OFFLINE, location=ce2cf0244921,33345,1731310155369; forceNewPlan=false, retain=false 2024-11-11T07:31:42,500 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=298c53e334bb69b7b36af68612f3d3c8, regionState=OPENING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:42,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=153, ppid=152, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=298c53e334bb69b7b36af68612f3d3c8, ASSIGN because future has completed 2024-11-11T07:31:42,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:31:42,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T07:31:42,656 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:42,657 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7752): Opening region: {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:31:42,657 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:42,657 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:31:42,657 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7794): checking encryption for 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:42,657 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(7797): checking classloading for 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:42,658 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:42,659 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:31:42,659 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 298c53e334bb69b7b36af68612f3d3c8 columnFamilyName A 2024-11-11T07:31:42,659 DEBUG [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:42,660 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.HStore(327): Store=298c53e334bb69b7b36af68612f3d3c8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:31:42,660 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:42,661 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:31:42,661 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 298c53e334bb69b7b36af68612f3d3c8 columnFamilyName B 2024-11-11T07:31:42,661 DEBUG [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:42,661 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.HStore(327): Store=298c53e334bb69b7b36af68612f3d3c8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:31:42,661 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:42,662 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:31:42,662 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 298c53e334bb69b7b36af68612f3d3c8 columnFamilyName C 2024-11-11T07:31:42,662 DEBUG [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:42,663 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.HStore(327): Store=298c53e334bb69b7b36af68612f3d3c8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:31:42,663 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:42,663 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1038): replaying wal for 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:42,663 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:42,664 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:42,664 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1048): stopping wal replay for 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:42,664 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1060): Cleaning up temporary data for 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:42,665 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:31:42,666 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1093): writing seq id for 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:42,667 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T07:31:42,668 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1114): Opened 298c53e334bb69b7b36af68612f3d3c8; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69657932, jitterRate=0.03798407316207886}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:31:42,668 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:42,668 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegion(1006): Region open journal for 298c53e334bb69b7b36af68612f3d3c8: Running coprocessor pre-open hook at 1731310302657Writing region info on filesystem at 1731310302657Initializing all the Stores at 1731310302658 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310302658Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310302658Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310302658Cleaning up temporary data from old regions at 1731310302664 (+6 ms)Running coprocessor post-open hooks at 1731310302668 (+4 ms)Region opened successfully at 1731310302668 2024-11-11T07:31:42,669 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., pid=154, masterSystemTime=1731310302654 2024-11-11T07:31:42,671 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:42,671 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=154}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:42,671 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=153 updating hbase:meta row=298c53e334bb69b7b36af68612f3d3c8, regionState=OPEN, openSeqNum=2, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:42,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=154, ppid=153, state=RUNNABLE, hasLock=false; OpenRegionProcedure 298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:31:42,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-11-11T07:31:42,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; OpenRegionProcedure 298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 in 171 msec 2024-11-11T07:31:42,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-11T07:31:42,676 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=298c53e334bb69b7b36af68612f3d3c8, ASSIGN in 328 msec 2024-11-11T07:31:42,677 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T07:31:42,677 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310302677"}]},"ts":"1731310302677"} 2024-11-11T07:31:42,678 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-11T07:31:42,679 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=152, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T07:31:42,680 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 751 msec 2024-11-11T07:31:43,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-11T07:31:43,065 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:43,066 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a769332, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:43,066 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:43,066 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:43,067 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22da96d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:43,067 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:43,068 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:43,068 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40742, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:43,069 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63c06902, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:43,069 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:43,070 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:43,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54462, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:43,072 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-11T07:31:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is ce2cf0244921,33029,1731310154427 2024-11-11T07:31:43,072 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6f3e5ef7 2024-11-11T07:31:43,073 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T07:31:43,073 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T07:31:43,075 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-11T07:31:43,075 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T07:31:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=155, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-11T07:31:43,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742474_1650 (size=1097) 2024-11-11T07:31:43,335 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T07:31:43,485 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-11T07:31:43,485 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-11T07:31:43,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:31:43,489 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=298c53e334bb69b7b36af68612f3d3c8, REOPEN/MOVE}] 2024-11-11T07:31:43,490 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=298c53e334bb69b7b36af68612f3d3c8, REOPEN/MOVE 2024-11-11T07:31:43,491 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=298c53e334bb69b7b36af68612f3d3c8, regionState=CLOSING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:43,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=298c53e334bb69b7b36af68612f3d3c8, REOPEN/MOVE because future has completed 2024-11-11T07:31:43,492 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:31:43,492 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure 298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:31:43,644 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(122): Close 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:43,644 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:31:43,644 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1722): Closing 298c53e334bb69b7b36af68612f3d3c8, disabling compactions & flushes 2024-11-11T07:31:43,644 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:43,644 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:43,644 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. after waiting 0 ms 2024-11-11T07:31:43,644 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:43,647 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-11T07:31:43,648 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:43,648 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegion(1676): Region close journal for 298c53e334bb69b7b36af68612f3d3c8: Waiting for close lock at 1731310303644Running coprocessor pre-close hooks at 1731310303644Disabling compacts and flushes for region at 1731310303644Disabling writes for close at 1731310303644Writing region close event to WAL at 1731310303645 (+1 ms)Running coprocessor post-close hooks at 1731310303648 (+3 ms)Closed at 1731310303648 2024-11-11T07:31:43,648 WARN [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] regionserver.HRegionServer(3299): Not adding moved region record: 298c53e334bb69b7b36af68612f3d3c8 to self. 2024-11-11T07:31:43,650 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=158}] handler.UnassignRegionHandler(157): Closed 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:43,651 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=298c53e334bb69b7b36af68612f3d3c8, regionState=CLOSED 2024-11-11T07:31:43,652 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=158, ppid=157, state=RUNNABLE, hasLock=false; CloseRegionProcedure 298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:31:43,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=158, resume processing ppid=157 2024-11-11T07:31:43,654 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, ppid=157, state=SUCCESS, hasLock=false; CloseRegionProcedure 298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 in 161 msec 2024-11-11T07:31:43,655 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=298c53e334bb69b7b36af68612f3d3c8, REOPEN/MOVE; state=CLOSED, location=ce2cf0244921,33345,1731310155369; forceNewPlan=false, retain=true 2024-11-11T07:31:43,805 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=298c53e334bb69b7b36af68612f3d3c8, regionState=OPENING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:43,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=157, ppid=156, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=298c53e334bb69b7b36af68612f3d3c8, REOPEN/MOVE because future has completed 2024-11-11T07:31:43,807 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure 298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:31:43,960 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:43,961 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7752): Opening region: {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} 2024-11-11T07:31:43,961 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:43,961 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T07:31:43,961 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7794): checking encryption for 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:43,961 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(7797): checking classloading for 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:43,962 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:43,963 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:31:43,963 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 298c53e334bb69b7b36af68612f3d3c8 columnFamilyName A 2024-11-11T07:31:43,964 DEBUG [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:43,964 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.HStore(327): Store=298c53e334bb69b7b36af68612f3d3c8/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:31:43,965 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:43,965 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:31:43,965 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 298c53e334bb69b7b36af68612f3d3c8 columnFamilyName B 2024-11-11T07:31:43,965 DEBUG [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:43,966 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.HStore(327): Store=298c53e334bb69b7b36af68612f3d3c8/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:31:43,966 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:43,966 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-11T07:31:43,966 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 298c53e334bb69b7b36af68612f3d3c8 columnFamilyName C 2024-11-11T07:31:43,967 DEBUG [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:43,967 INFO [StoreOpener-298c53e334bb69b7b36af68612f3d3c8-1 {}] regionserver.HStore(327): Store=298c53e334bb69b7b36af68612f3d3c8/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T07:31:43,967 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:43,967 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1038): replaying wal for 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:43,968 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:43,968 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:43,969 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1048): stopping wal replay for 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:43,969 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1060): Cleaning up temporary data for 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:43,969 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-11T07:31:43,970 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1093): writing seq id for 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:43,971 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1114): Opened 298c53e334bb69b7b36af68612f3d3c8; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75064956, jitterRate=0.11855500936508179}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-11T07:31:43,971 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:43,972 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegion(1006): Region open journal for 298c53e334bb69b7b36af68612f3d3c8: Running coprocessor pre-open hook at 1731310303961Writing region info on filesystem at 1731310303961Initializing all the Stores at 1731310303962 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310303962Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310303962Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731310303962Cleaning up temporary data from old regions at 1731310303969 (+7 ms)Running coprocessor post-open hooks at 1731310303971 (+2 ms)Region opened successfully at 1731310303972 (+1 ms) 2024-11-11T07:31:43,972 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., pid=159, masterSystemTime=1731310303958 2024-11-11T07:31:43,974 DEBUG [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:43,974 INFO [RS_OPEN_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_OPEN_REGION, pid=159}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:43,974 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=157 updating hbase:meta row=298c53e334bb69b7b36af68612f3d3c8, regionState=OPEN, openSeqNum=5, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:43,976 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=159, ppid=157, state=RUNNABLE, hasLock=false; OpenRegionProcedure 298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:31:43,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=157 2024-11-11T07:31:43,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=157, state=SUCCESS, hasLock=false; OpenRegionProcedure 298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 in 169 msec 2024-11-11T07:31:43,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-11T07:31:43,979 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=298c53e334bb69b7b36af68612f3d3c8, REOPEN/MOVE in 489 msec 2024-11-11T07:31:43,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=156, resume processing ppid=155 2024-11-11T07:31:43,982 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, ppid=155, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 493 msec 2024-11-11T07:31:43,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 906 msec 2024-11-11T07:31:43,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-11-11T07:31:43,984 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:43,985 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bd32fa2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:43,985 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:43,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:43,986 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@db6802d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:43,986 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:43,987 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:43,987 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:43,988 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4906b96b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:43,988 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d29c9e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:43,988 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:43,988 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:43,989 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:43,989 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16838c15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:43,990 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:43,990 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40788, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:43,991 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4779844a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:43,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1572f078, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:43,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:43,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:43,993 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@123b082b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:43,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:43,994 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:43,994 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37b9d23a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:43,995 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23448b6e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:43,995 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:43,995 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:43,996 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6883c7b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:43,996 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:43,997 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:43,997 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40818, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:43,998 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44627c7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:43,998 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d7db06e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:43,998 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:43,999 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:43,999 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:43,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@48c32b33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:44,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:44,001 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40836, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:44,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@411f65ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41947491, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,002 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:44,002 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:44,002 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@710a4d14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:44,003 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:44,004 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40854, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:44,004 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e78b29c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,005 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5451f0cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,005 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:44,005 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:44,006 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1297acbd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:44,006 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:44,007 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40878, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:44,007 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f2faf33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b0e0fb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:44,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:44,009 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f18199d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,009 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:44,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:44,010 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40904, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:44,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f84f007, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,012 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbb8a41, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,012 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:44,012 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:44,012 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b781ae1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:44,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:44,014 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40932, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:44,014 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@288c6b99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,015 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31a58f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,015 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request ce2cf0244921,33029,-1 for getting cluster id 2024-11-11T07:31:44,015 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-11T07:31:44,016 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a1f130b9-1abe-4225-8bb4-570f54060737' 2024-11-11T07:31:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-11T07:31:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a1f130b9-1abe-4225-8bb4-570f54060737" 2024-11-11T07:31:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e046b83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [ce2cf0244921,33029,-1] 2024-11-11T07:31:44,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-11T07:31:44,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:31:44,017 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40952, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-11T07:31:44,018 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b94913d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T07:31:44,019 DEBUG [Thread-2858 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:44,019 DEBUG [Thread-2860 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:44,019 DEBUG [Thread-2851 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:44,020 DEBUG [Thread-2843 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:44,020 DEBUG [Thread-2851 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:44,020 DEBUG [Thread-2851 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:44,020 DEBUG [Thread-2849 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:44,020 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:44,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54488, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:44,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:44,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:44,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:44,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:44,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54518, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:44,022 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:44,023 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:44,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:31:44,023 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54480, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:44,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:31:44,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:31:44,025 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:31:44,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:31:44,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:44,036 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T07:31:44,039 DEBUG [Thread-2845 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:44,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:31:44,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:44,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:31:44,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:44,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:31:44,039 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:44,040 DEBUG [Thread-2845 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:44,040 DEBUG [Thread-2845 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:44,041 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54524, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:44,043 DEBUG [Thread-2845 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:31:44,047 DEBUG [Thread-2854 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:44,048 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:44,049 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:44,051 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:31:44,065 DEBUG [Thread-2847 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:44,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310304056, max timestamp=1731310304064Num uniques 859; 2024-11-11T07:31:44,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310304056, max timestamp=1731310304064Num uniques 859; 2024-11-11T07:31:44,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:31:44,067 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:31:44,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310304056, max timestamp=1731310304066Num uniques 859; 2024-11-11T07:31:44,069 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:31:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:44,070 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:44,072 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54550, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:44,073 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:31:44,077 DEBUG [Thread-2856 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:44,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:44,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-11T07:31:44,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:44,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T07:31:44,081 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:44,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:44,082 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:44,083 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:44,083 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:44,084 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:31:44,095 DEBUG [Thread-2862 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-11T07:31:44,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310364090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,099 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310364090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,100 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54550 deadline: 1731310364092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,100 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,101 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,102 DEBUG [Thread-2862 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=ce2cf0244921,33345,1731310155369, seqNum=-1] 2024-11-11T07:31:44,102 DEBUG [Thread-2862 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T07:31:44,103 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T07:31:44,105 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5] 2024-11-11T07:31:44,106 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111187250d69f1d4332ab3354e835bb3152_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310304035/Put/seqid=0 2024-11-11T07:31:44,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310364099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,107 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731310364100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,108 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742475_1651 (size=14594) 2024-11-11T07:31:44,115 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:44,119 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111187250d69f1d4332ab3354e835bb3152_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111187250d69f1d4332ab3354e835bb3152_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:44,121 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/afc88d82b1944f29b36e51364cfbbb8f, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:44,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/afc88d82b1944f29b36e51364cfbbb8f is 175, key is test_row_0/A:col10/1731310304035/Put/seqid=0 2024-11-11T07:31:44,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742476_1652 (size=39549) 2024-11-11T07:31:44,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T07:31:44,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310364205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,209 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,209 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310364205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54550 deadline: 1731310364205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,210 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,216 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310364214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,217 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731310364214, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,217 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,218 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T07:31:44,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:44,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:44,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:44,239 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:44,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:44,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T07:31:44,399 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T07:31:44,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:44,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:44,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:44,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:44,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:44,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:44,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310364415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54550 deadline: 1731310364415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,418 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310364419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,426 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310364424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731310364424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,552 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/afc88d82b1944f29b36e51364cfbbb8f 2024-11-11T07:31:44,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T07:31:44,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:44,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:44,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:44,599 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:44,599 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:44,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:44,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/283c2be50f47494d9ddc55e4fb267e6f is 50, key is test_row_0/B:col10/1731310304035/Put/seqid=0 2024-11-11T07:31:44,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742477_1653 (size=12001) 2024-11-11T07:31:44,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T07:31:44,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310364724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54550 deadline: 1731310364724, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,730 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,739 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310364739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,757 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731310364756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,758 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,759 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:44,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310364756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:44,759 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:44,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T07:31:44,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:44,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:44,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:44,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:44,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:44,927 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T07:31:44,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:44,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:44,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:44,928 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:44,928 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:44,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:45,059 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/283c2be50f47494d9ddc55e4fb267e6f 2024-11-11T07:31:45,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T07:31:45,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:45,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:45,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:45,084 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:45,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:45,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:45,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/becc423616964b78a4edce00e75917f7 is 50, key is test_row_0/C:col10/1731310304035/Put/seqid=0 2024-11-11T07:31:45,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742478_1654 (size=12001) 2024-11-11T07:31:45,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T07:31:45,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:45,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54550 deadline: 1731310365234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:45,238 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:45,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T07:31:45,240 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:45,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:45,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:45,241 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:45,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:45,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:45,257 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:45,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310365255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:45,257 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:45,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:45,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731310365264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:45,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:45,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:45,267 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:45,268 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:45,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310365264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:45,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:45,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:45,270 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:45,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310365266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:45,274 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:45,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:45,395 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T07:31:45,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:45,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:45,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:45,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:45,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:45,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:45,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/becc423616964b78a4edce00e75917f7 2024-11-11T07:31:45,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/afc88d82b1944f29b36e51364cfbbb8f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/afc88d82b1944f29b36e51364cfbbb8f 2024-11-11T07:31:45,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/afc88d82b1944f29b36e51364cfbbb8f, entries=200, sequenceid=15, filesize=38.6 K 2024-11-11T07:31:45,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/283c2be50f47494d9ddc55e4fb267e6f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/283c2be50f47494d9ddc55e4fb267e6f 2024-11-11T07:31:45,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/283c2be50f47494d9ddc55e4fb267e6f, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T07:31:45,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/becc423616964b78a4edce00e75917f7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/becc423616964b78a4edce00e75917f7 2024-11-11T07:31:45,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T07:31:45,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:45,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:45,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:45,548 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:45,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:45,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:45,555 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/becc423616964b78a4edce00e75917f7, entries=150, sequenceid=15, filesize=11.7 K 2024-11-11T07:31:45,559 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 298c53e334bb69b7b36af68612f3d3c8 in 1524ms, sequenceid=15, compaction requested=false 2024-11-11T07:31:45,559 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-11T07:31:45,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:45,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-11T07:31:45,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:45,707 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=127.47 KB heapSize=365.06 KB 2024-11-11T07:31:45,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:31:45,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:45,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:31:45,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:45,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:31:45,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:45,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f470154db25f411a9465b5ed814c27df_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310304092/Put/seqid=0 2024-11-11T07:31:45,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742479_1655 (size=12154) 2024-11-11T07:31:46,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:46,160 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f470154db25f411a9465b5ed814c27df_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f470154db25f411a9465b5ed814c27df_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:46,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/a82f8dc1a36f471eb3ec0d1b17a3aefa, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:46,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/a82f8dc1a36f471eb3ec0d1b17a3aefa is 175, key is test_row_0/A:col10/1731310304092/Put/seqid=0 2024-11-11T07:31:46,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742480_1656 (size=30955) 2024-11-11T07:31:46,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T07:31:46,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:46,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:46,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310366275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:46,282 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_2' in 298c53e334bb69b7b36af68612f3d3c8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2191 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:31:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:46,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:46,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54550 deadline: 1731310366281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:46,286 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:46,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731310366283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,286 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:46,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:46,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310366284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:46,288 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_1' in 298c53e334bb69b7b36af68612f3d3c8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:31:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,289 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:46,292 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:46,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310366286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:46,294 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_0' in 298c53e334bb69b7b36af68612f3d3c8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:31:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:46,308 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T07:31:46,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54550 deadline: 1731310366395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:46,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:46,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731310366395, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:46,398 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,399 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:46,566 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/a82f8dc1a36f471eb3ec0d1b17a3aefa 2024-11-11T07:31:46,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/d87991db832c473b81feb3aa1b28b34d is 50, key is test_row_0/B:col10/1731310304092/Put/seqid=0 2024-11-11T07:31:46,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742481_1657 (size=12001) 2024-11-11T07:31:46,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731310366605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:46,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:46,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54550 deadline: 1731310366605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:46,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,610 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:46,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,610 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:46,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:46,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54550 deadline: 1731310366915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731310366915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:46,919 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:47,004 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/d87991db832c473b81feb3aa1b28b34d 2024-11-11T07:31:47,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/0756f8fb796944339a0714f842949054 is 50, key is test_row_0/C:col10/1731310304092/Put/seqid=0 2024-11-11T07:31:47,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742482_1658 (size=12001) 2024-11-11T07:31:47,426 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:47,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54550 deadline: 1731310367424, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:47,427 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:47,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:47,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731310367434, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:47,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:47,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:47,439 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:47,455 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/0756f8fb796944339a0714f842949054 2024-11-11T07:31:47,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/a82f8dc1a36f471eb3ec0d1b17a3aefa as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/a82f8dc1a36f471eb3ec0d1b17a3aefa 2024-11-11T07:31:47,466 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/a82f8dc1a36f471eb3ec0d1b17a3aefa, entries=150, sequenceid=51, filesize=30.2 K 2024-11-11T07:31:47,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/d87991db832c473b81feb3aa1b28b34d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/d87991db832c473b81feb3aa1b28b34d 2024-11-11T07:31:47,470 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/d87991db832c473b81feb3aa1b28b34d, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T07:31:47,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/0756f8fb796944339a0714f842949054 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/0756f8fb796944339a0714f842949054 2024-11-11T07:31:47,474 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/0756f8fb796944339a0714f842949054, entries=150, sequenceid=51, filesize=11.7 K 2024-11-11T07:31:47,475 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~365.02 KB/373776, currentSize=53.67 KB/54960 for 298c53e334bb69b7b36af68612f3d3c8 in 1767ms, sequenceid=51, compaction requested=false 2024-11-11T07:31:47,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:47,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:47,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-11T07:31:47,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-11T07:31:47,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-11T07:31:47,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3960 sec 2024-11-11T07:31:47,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 3.4160 sec 2024-11-11T07:31:48,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-11T07:31:48,235 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:48,236 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:48,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-11T07:31:48,237 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T07:31:48,238 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:48,238 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:48,301 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-11T07:31:48,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:31:48,301 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:48,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:31:48,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:48,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:31:48,302 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:48,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:48,331 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111eaa02b3273ad43da8ec73cc5701ea77d_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310306263/Put/seqid=0 2024-11-11T07:31:48,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742483_1659 (size=12154) 2024-11-11T07:31:48,338 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:48,343 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111eaa02b3273ad43da8ec73cc5701ea77d_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111eaa02b3273ad43da8ec73cc5701ea77d_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:48,344 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/390146f70796429fa23f47f34bfb880f, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:48,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T07:31:48,344 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/390146f70796429fa23f47f34bfb880f is 175, key is test_row_0/A:col10/1731310306263/Put/seqid=0 2024-11-11T07:31:48,369 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310368362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:48,369 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:48,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742484_1660 (size=30955) 2024-11-11T07:31:48,372 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=63, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/390146f70796429fa23f47f34bfb880f 2024-11-11T07:31:48,372 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:48,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310368365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:48,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,372 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:48,378 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:48,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310368369, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:48,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/2a1660a4b3de45278271a2322faee91e is 50, key is test_row_0/B:col10/1731310306263/Put/seqid=0 2024-11-11T07:31:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:48,390 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T07:31:48,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:48,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:48,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:48,390 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:48,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:48,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:48,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742485_1661 (size=12001) 2024-11-11T07:31:48,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:48,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54550 deadline: 1731310368435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:48,439 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_1' in 298c53e334bb69b7b36af68612f3d3c8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:31:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,440 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:48,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:48,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731310368456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:48,460 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_1' in 298c53e334bb69b7b36af68612f3d3c8 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:31:48,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,461 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:48,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:48,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310368474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:48,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310368475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:48,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:48,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310368485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:48,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T07:31:48,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:48,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:48,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:48,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:48,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:48,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T07:31:48,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:48,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310368685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:48,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310368685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,689 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:48,696 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:48,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310368694, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,697 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:48,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T07:31:48,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:48,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:48,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:48,706 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:48,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:48,819 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/2a1660a4b3de45278271a2322faee91e 2024-11-11T07:31:48,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T07:31:48,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T07:31:48,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/74fa0ce706c54c18911a82d90feacd5e is 50, key is test_row_0/C:col10/1731310306263/Put/seqid=0 2024-11-11T07:31:48,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:48,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:48,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:48,870 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:48,870 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:48,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:48,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742486_1662 (size=12001) 2024-11-11T07:31:48,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:48,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310368995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,997 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:48,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:48,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310368995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:48,999 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:49,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:49,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310369004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:49,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T07:31:49,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:49,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:49,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:49,026 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:49,026 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:49,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:49,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T07:31:49,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:49,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:49,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:49,183 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:49,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:49,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:49,289 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/74fa0ce706c54c18911a82d90feacd5e 2024-11-11T07:31:49,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/390146f70796429fa23f47f34bfb880f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/390146f70796429fa23f47f34bfb880f 2024-11-11T07:31:49,299 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/390146f70796429fa23f47f34bfb880f, entries=150, sequenceid=63, filesize=30.2 K 2024-11-11T07:31:49,300 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/2a1660a4b3de45278271a2322faee91e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/2a1660a4b3de45278271a2322faee91e 2024-11-11T07:31:49,303 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/2a1660a4b3de45278271a2322faee91e, entries=150, sequenceid=63, filesize=11.7 K 2024-11-11T07:31:49,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/74fa0ce706c54c18911a82d90feacd5e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/74fa0ce706c54c18911a82d90feacd5e 2024-11-11T07:31:49,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/74fa0ce706c54c18911a82d90feacd5e, entries=150, sequenceid=63, filesize=11.7 K 2024-11-11T07:31:49,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 298c53e334bb69b7b36af68612f3d3c8 in 1019ms, sequenceid=63, compaction requested=true 2024-11-11T07:31:49,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:49,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:49,320 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:49,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:49,320 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:49,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:49,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:49,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:49,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:49,321 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:49,321 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:49,321 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/A is initiating minor compaction (all files) 2024-11-11T07:31:49,321 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/B is initiating minor compaction (all files) 2024-11-11T07:31:49,321 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/A in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:49,321 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/B in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:49,322 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/283c2be50f47494d9ddc55e4fb267e6f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/d87991db832c473b81feb3aa1b28b34d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/2a1660a4b3de45278271a2322faee91e] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=35.2 K 2024-11-11T07:31:49,322 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/afc88d82b1944f29b36e51364cfbbb8f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/a82f8dc1a36f471eb3ec0d1b17a3aefa, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/390146f70796429fa23f47f34bfb880f] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=99.1 K 2024-11-11T07:31:49,322 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:49,322 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/afc88d82b1944f29b36e51364cfbbb8f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/a82f8dc1a36f471eb3ec0d1b17a3aefa, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/390146f70796429fa23f47f34bfb880f] 2024-11-11T07:31:49,322 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 283c2be50f47494d9ddc55e4fb267e6f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731310304031 2024-11-11T07:31:49,322 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting afc88d82b1944f29b36e51364cfbbb8f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731310304031 2024-11-11T07:31:49,322 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting a82f8dc1a36f471eb3ec0d1b17a3aefa, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731310304090 2024-11-11T07:31:49,322 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting d87991db832c473b81feb3aa1b28b34d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731310304090 2024-11-11T07:31:49,323 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 390146f70796429fa23f47f34bfb880f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731310306263 2024-11-11T07:31:49,323 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a1660a4b3de45278271a2322faee91e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731310306263 2024-11-11T07:31:49,332 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#B#compaction#570 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:49,333 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/494a54ea89dd4735aa7d45a65aa70e01 is 50, key is test_row_0/B:col10/1731310306263/Put/seqid=0 2024-11-11T07:31:49,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-11T07:31:49,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:49,342 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:31:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:31:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:31:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:31:49,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:49,346 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:49,368 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111c4f126c2b30a43ad82d7a65bc199a222_298c53e334bb69b7b36af68612f3d3c8 store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:49,370 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111c4f126c2b30a43ad82d7a65bc199a222_298c53e334bb69b7b36af68612f3d3c8, store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:49,371 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111c4f126c2b30a43ad82d7a65bc199a222_298c53e334bb69b7b36af68612f3d3c8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:49,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d637c7a3bc754cbdb51218eaf9c4b608_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310308364/Put/seqid=0 2024-11-11T07:31:49,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T07:31:49,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742487_1663 (size=12104) 2024-11-11T07:31:49,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742488_1664 (size=4469) 2024-11-11T07:31:49,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742489_1665 (size=12154) 2024-11-11T07:31:49,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:49,507 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111d637c7a3bc754cbdb51218eaf9c4b608_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d637c7a3bc754cbdb51218eaf9c4b608_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/8743ba9ca75948219b43bbbabcef7f6b, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:49,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/8743ba9ca75948219b43bbbabcef7f6b is 175, key is test_row_0/A:col10/1731310308364/Put/seqid=0 2024-11-11T07:31:49,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310369529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,531 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:49,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310369530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:49,536 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:49,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310369531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:49,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,536 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:49,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,537 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:49,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742490_1666 (size=30955) 2024-11-11T07:31:49,543 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=83, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/8743ba9ca75948219b43bbbabcef7f6b 2024-11-11T07:31:49,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/b884fc5936e14dd18e5ce788f5037858 is 50, key is test_row_0/B:col10/1731310308364/Put/seqid=0 2024-11-11T07:31:49,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742491_1667 (size=12001) 2024-11-11T07:31:49,576 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/b884fc5936e14dd18e5ce788f5037858 2024-11-11T07:31:49,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/c5bcfb089ed546899cf4ec599cbd3ead is 50, key is test_row_0/C:col10/1731310308364/Put/seqid=0 2024-11-11T07:31:49,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742492_1668 (size=12001) 2024-11-11T07:31:49,613 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=83 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/c5bcfb089ed546899cf4ec599cbd3ead 2024-11-11T07:31:49,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/8743ba9ca75948219b43bbbabcef7f6b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/8743ba9ca75948219b43bbbabcef7f6b 2024-11-11T07:31:49,621 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/8743ba9ca75948219b43bbbabcef7f6b, entries=150, sequenceid=83, filesize=30.2 K 2024-11-11T07:31:49,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/b884fc5936e14dd18e5ce788f5037858 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/b884fc5936e14dd18e5ce788f5037858 2024-11-11T07:31:49,628 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/b884fc5936e14dd18e5ce788f5037858, entries=150, sequenceid=83, filesize=11.7 K 2024-11-11T07:31:49,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/c5bcfb089ed546899cf4ec599cbd3ead as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/c5bcfb089ed546899cf4ec599cbd3ead 2024-11-11T07:31:49,633 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/c5bcfb089ed546899cf4ec599cbd3ead, entries=150, sequenceid=83, filesize=11.7 K 2024-11-11T07:31:49,634 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 298c53e334bb69b7b36af68612f3d3c8 in 293ms, sequenceid=83, compaction requested=true 2024-11-11T07:31:49,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:49,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:49,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-11T07:31:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-11T07:31:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:49,638 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T07:31:49,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:31:49,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:49,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:31:49,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:49,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:31:49,638 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:49,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-11T07:31:49,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3990 sec 2024-11-11T07:31:49,647 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 1.4090 sec 2024-11-11T07:31:49,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118199e9c9ff7041e491d2677d905d4ae3_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310309636/Put/seqid=0 2024-11-11T07:31:49,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742493_1669 (size=14594) 2024-11-11T07:31:49,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:49,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310369689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310369690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:49,698 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310369691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:49,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,700 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:49,796 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/494a54ea89dd4735aa7d45a65aa70e01 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/494a54ea89dd4735aa7d45a65aa70e01 2024-11-11T07:31:49,800 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/B of 298c53e334bb69b7b36af68612f3d3c8 into 494a54ea89dd4735aa7d45a65aa70e01(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:49,800 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:49,800 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/B, priority=13, startTime=1731310309320; duration=0sec 2024-11-11T07:31:49,801 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:49,801 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:B 2024-11-11T07:31:49,801 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:49,802 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:49,802 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/C is initiating minor compaction (all files) 2024-11-11T07:31:49,802 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/C in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:49,802 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/becc423616964b78a4edce00e75917f7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/0756f8fb796944339a0714f842949054, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/74fa0ce706c54c18911a82d90feacd5e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/c5bcfb089ed546899cf4ec599cbd3ead] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=46.9 K 2024-11-11T07:31:49,802 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting becc423616964b78a4edce00e75917f7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731310304031 2024-11-11T07:31:49,802 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 0756f8fb796944339a0714f842949054, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731310304090 2024-11-11T07:31:49,802 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 74fa0ce706c54c18911a82d90feacd5e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731310306263 2024-11-11T07:31:49,803 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting c5bcfb089ed546899cf4ec599cbd3ead, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731310308361 2024-11-11T07:31:49,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310369804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,805 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310369804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:49,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310369804, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,805 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:49,806 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:49,811 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#C#compaction#576 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:49,811 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/e50386f66aff4a868b9a06921d2df151 is 50, key is test_row_0/C:col10/1731310308364/Put/seqid=0 2024-11-11T07:31:49,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742494_1670 (size=12139) 2024-11-11T07:31:49,854 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/e50386f66aff4a868b9a06921d2df151 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/e50386f66aff4a868b9a06921d2df151 2024-11-11T07:31:49,869 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/C of 298c53e334bb69b7b36af68612f3d3c8 into e50386f66aff4a868b9a06921d2df151(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:49,869 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:49,869 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/C, priority=12, startTime=1731310309321; duration=0sec 2024-11-11T07:31:49,869 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:49,869 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:C 2024-11-11T07:31:49,877 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#A#compaction#571 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:49,877 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/5a1b6fa78a7f4c588db530b9d1f0f757 is 175, key is test_row_0/A:col10/1731310306263/Put/seqid=0 2024-11-11T07:31:49,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742495_1671 (size=31058) 2024-11-11T07:31:50,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:50,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310370015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:50,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:50,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310370015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310370015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:50,019 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:50,076 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:50,079 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411118199e9c9ff7041e491d2677d905d4ae3_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118199e9c9ff7041e491d2677d905d4ae3_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:50,080 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/e1a218f0c24d40f78ad36d9a52dd54cd, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:50,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/e1a218f0c24d40f78ad36d9a52dd54cd is 175, key is test_row_0/A:col10/1731310309636/Put/seqid=0 2024-11-11T07:31:50,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742496_1672 (size=39549) 2024-11-11T07:31:50,306 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/5a1b6fa78a7f4c588db530b9d1f0f757 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/5a1b6fa78a7f4c588db530b9d1f0f757 2024-11-11T07:31:50,313 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/A of 298c53e334bb69b7b36af68612f3d3c8 into 5a1b6fa78a7f4c588db530b9d1f0f757(size=30.3 K), total size for store is 60.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:50,313 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:50,313 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/A, priority=13, startTime=1731310309320; duration=0sec 2024-11-11T07:31:50,313 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:50,313 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:A 2024-11-11T07:31:50,327 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:50,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310370324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:50,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:50,328 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:50,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310370324, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:50,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:50,328 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:50,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310370325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:50,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:50,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-11T07:31:50,384 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:50,385 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:50,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-11T07:31:50,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T07:31:50,394 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:50,395 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:50,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:50,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:50,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54550 deadline: 1731310370445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:50,448 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_1' in 298c53e334bb69b7b36af68612f3d3c8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:31:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,449 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:50,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:50,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731310370484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:50,487 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_1' in 298c53e334bb69b7b36af68612f3d3c8 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:31:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:50,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T07:31:50,510 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=96, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/e1a218f0c24d40f78ad36d9a52dd54cd 2024-11-11T07:31:50,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/e3d621e362324e519d39cdd6a4248bc1 is 50, key is test_row_0/B:col10/1731310309636/Put/seqid=0 2024-11-11T07:31:50,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-11T07:31:50,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:50,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:50,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:50,551 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:50,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:50,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:50,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742497_1673 (size=12001) 2024-11-11T07:31:50,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/e3d621e362324e519d39cdd6a4248bc1 2024-11-11T07:31:50,590 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/994d488d826741dcaabdac9db56047d2 is 50, key is test_row_0/C:col10/1731310309636/Put/seqid=0 2024-11-11T07:31:50,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742498_1674 (size=12001) 2024-11-11T07:31:50,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=96 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/994d488d826741dcaabdac9db56047d2 2024-11-11T07:31:50,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/e1a218f0c24d40f78ad36d9a52dd54cd as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/e1a218f0c24d40f78ad36d9a52dd54cd 2024-11-11T07:31:50,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/e1a218f0c24d40f78ad36d9a52dd54cd, entries=200, sequenceid=96, filesize=38.6 K 2024-11-11T07:31:50,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/e3d621e362324e519d39cdd6a4248bc1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/e3d621e362324e519d39cdd6a4248bc1 2024-11-11T07:31:50,645 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/e3d621e362324e519d39cdd6a4248bc1, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T07:31:50,646 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/994d488d826741dcaabdac9db56047d2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/994d488d826741dcaabdac9db56047d2 2024-11-11T07:31:50,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/994d488d826741dcaabdac9db56047d2, entries=150, sequenceid=96, filesize=11.7 K 2024-11-11T07:31:50,651 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 298c53e334bb69b7b36af68612f3d3c8 in 1013ms, sequenceid=96, compaction requested=true 2024-11-11T07:31:50,651 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:50,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:50,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:50,651 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:50,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:50,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:50,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:50,651 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:50,651 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:50,652 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:50,652 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:50,652 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/A is initiating minor compaction (all files) 2024-11-11T07:31:50,652 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/B is initiating minor compaction (all files) 2024-11-11T07:31:50,652 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/B in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:50,652 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/A in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:50,652 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/494a54ea89dd4735aa7d45a65aa70e01, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/b884fc5936e14dd18e5ce788f5037858, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/e3d621e362324e519d39cdd6a4248bc1] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=35.3 K 2024-11-11T07:31:50,652 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/5a1b6fa78a7f4c588db530b9d1f0f757, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/8743ba9ca75948219b43bbbabcef7f6b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/e1a218f0c24d40f78ad36d9a52dd54cd] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=99.2 K 2024-11-11T07:31:50,652 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:50,652 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/5a1b6fa78a7f4c588db530b9d1f0f757, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/8743ba9ca75948219b43bbbabcef7f6b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/e1a218f0c24d40f78ad36d9a52dd54cd] 2024-11-11T07:31:50,652 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 494a54ea89dd4735aa7d45a65aa70e01, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731310306263 2024-11-11T07:31:50,652 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a1b6fa78a7f4c588db530b9d1f0f757, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731310306263 2024-11-11T07:31:50,653 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting b884fc5936e14dd18e5ce788f5037858, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731310308361 2024-11-11T07:31:50,653 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8743ba9ca75948219b43bbbabcef7f6b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731310308361 2024-11-11T07:31:50,653 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting e1a218f0c24d40f78ad36d9a52dd54cd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731310309520 2024-11-11T07:31:50,653 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting e3d621e362324e519d39cdd6a4248bc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731310309527 2024-11-11T07:31:50,659 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#B#compaction#579 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:50,660 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/e06d60a9169e4e08a8ab2df24fa63f3b is 50, key is test_row_0/B:col10/1731310309636/Put/seqid=0 2024-11-11T07:31:50,661 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:50,667 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111601d2980153844ba858326e4aae362b8_298c53e334bb69b7b36af68612f3d3c8 store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:50,669 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111601d2980153844ba858326e4aae362b8_298c53e334bb69b7b36af68612f3d3c8, store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:50,669 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111601d2980153844ba858326e4aae362b8_298c53e334bb69b7b36af68612f3d3c8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:50,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742499_1675 (size=12207) 2024-11-11T07:31:50,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T07:31:50,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-11T07:31:50,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:50,715 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-11T07:31:50,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:31:50,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:50,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:31:50,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:50,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:31:50,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:50,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742500_1676 (size=4469) 2024-11-11T07:31:50,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fdf8b231b0d044009af732fdf3b5c6b4_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310309690/Put/seqid=0 2024-11-11T07:31:50,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742501_1677 (size=12154) 2024-11-11T07:31:50,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:50,786 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fdf8b231b0d044009af732fdf3b5c6b4_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fdf8b231b0d044009af732fdf3b5c6b4_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:50,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/c71e3465912b44c887587fc266ab8897, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:50,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/c71e3465912b44c887587fc266ab8897 is 175, key is test_row_0/A:col10/1731310309690/Put/seqid=0 2024-11-11T07:31:50,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742502_1678 (size=30955) 2024-11-11T07:31:50,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:50,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:50,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310370865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:50,874 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:50,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310370867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,874 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,875 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:50,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,875 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:50,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:50,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310370874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:50,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:50,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310370985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,988 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:50,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:50,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310370985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:50,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:50,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:50,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310370985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:50,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:50,990 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:51,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T07:31:51,099 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/e06d60a9169e4e08a8ab2df24fa63f3b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/e06d60a9169e4e08a8ab2df24fa63f3b 2024-11-11T07:31:51,105 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/B of 298c53e334bb69b7b36af68612f3d3c8 into e06d60a9169e4e08a8ab2df24fa63f3b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:51,105 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:51,105 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/B, priority=13, startTime=1731310310651; duration=0sec 2024-11-11T07:31:51,105 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:51,105 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:B 2024-11-11T07:31:51,105 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-11T07:31:51,106 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-11T07:31:51,106 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-11T07:31:51,106 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. because compaction request was cancelled 2024-11-11T07:31:51,106 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:C 2024-11-11T07:31:51,133 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#A#compaction#580 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:51,133 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/c559194535114d029ac7b56757081cfd is 175, key is test_row_0/A:col10/1731310309636/Put/seqid=0 2024-11-11T07:31:51,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742503_1679 (size=31161) 2024-11-11T07:31:51,198 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/c559194535114d029ac7b56757081cfd as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/c559194535114d029ac7b56757081cfd 2024-11-11T07:31:51,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310371195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:51,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:51,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310371199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:51,201 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:51,201 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:51,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310371199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:51,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:51,228 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/A of 298c53e334bb69b7b36af68612f3d3c8 into c559194535114d029ac7b56757081cfd(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:51,228 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:51,228 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/A, priority=13, startTime=1731310310651; duration=0sec 2024-11-11T07:31:51,228 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:51,228 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:A 2024-11-11T07:31:51,228 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/c71e3465912b44c887587fc266ab8897 2024-11-11T07:31:51,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/92b9f5d41bd2466583b12e77a28af41e is 50, key is test_row_0/B:col10/1731310309690/Put/seqid=0 2024-11-11T07:31:51,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742504_1680 (size=12001) 2024-11-11T07:31:51,248 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/92b9f5d41bd2466583b12e77a28af41e 2024-11-11T07:31:51,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/b238065298e644e0bbac8a6033ad1947 is 50, key is test_row_0/C:col10/1731310309690/Put/seqid=0 2024-11-11T07:31:51,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742505_1681 (size=12001) 2024-11-11T07:31:51,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:51,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310371504, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:51,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:51,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:51,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:51,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:51,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310371506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:51,510 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:51,511 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:51,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:51,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310371515, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:51,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:51,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:51,517 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:51,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T07:31:51,682 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/b238065298e644e0bbac8a6033ad1947 2024-11-11T07:31:51,688 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/c71e3465912b44c887587fc266ab8897 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/c71e3465912b44c887587fc266ab8897 2024-11-11T07:31:51,691 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/c71e3465912b44c887587fc266ab8897, entries=150, sequenceid=118, filesize=30.2 K 2024-11-11T07:31:51,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/92b9f5d41bd2466583b12e77a28af41e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/92b9f5d41bd2466583b12e77a28af41e 2024-11-11T07:31:51,697 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/92b9f5d41bd2466583b12e77a28af41e, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T07:31:51,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/b238065298e644e0bbac8a6033ad1947 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/b238065298e644e0bbac8a6033ad1947 2024-11-11T07:31:51,703 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/b238065298e644e0bbac8a6033ad1947, entries=150, sequenceid=118, filesize=11.7 K 2024-11-11T07:31:51,704 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 298c53e334bb69b7b36af68612f3d3c8 in 989ms, sequenceid=118, compaction requested=true 2024-11-11T07:31:51,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:51,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:51,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-11T07:31:51,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-11T07:31:51,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-11T07:31:51,707 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3110 sec 2024-11-11T07:31:51,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 1.3220 sec 2024-11-11T07:31:52,017 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-11T07:31:52,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:52,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:31:52,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:52,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:31:52,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:52,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:31:52,019 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:52,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f253823c12a44df7929f2e5b0e5fc8f1_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310310851/Put/seqid=0 2024-11-11T07:31:52,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:52,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310372046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:52,054 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310372049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:52,055 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310372053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,057 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:52,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742506_1682 (size=14744) 2024-11-11T07:31:52,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310372164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:52,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310372164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:52,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310372165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:52,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,169 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:52,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,169 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:52,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310372375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:52,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310372375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310372375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:52,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:52,490 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:52,493 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f253823c12a44df7929f2e5b0e5fc8f1_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f253823c12a44df7929f2e5b0e5fc8f1_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:52,493 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/73f342402564428c84a9a0f2aea8995c, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:52,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/73f342402564428c84a9a0f2aea8995c is 175, key is test_row_0/A:col10/1731310310851/Put/seqid=0 2024-11-11T07:31:52,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-11T07:31:52,535 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:52,535 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-11T07:31:52,538 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:52,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T07:31:52,539 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:52,539 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:52,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742507_1683 (size=39699) 2024-11-11T07:31:52,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T07:31:52,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:52,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310372685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:52,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310372685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,688 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:52,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310372685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:52,688 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:52,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T07:31:52,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:52,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:52,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:52,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:52,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:52,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:52,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T07:31:52,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:52,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:52,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:52,854 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:52,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:52,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:52,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T07:31:52,953 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/73f342402564428c84a9a0f2aea8995c 2024-11-11T07:31:52,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/01543f79bca6463fbf8b8ecad780bee6 is 50, key is test_row_0/B:col10/1731310310851/Put/seqid=0 2024-11-11T07:31:53,006 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742508_1684 (size=12101) 2024-11-11T07:31:53,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/01543f79bca6463fbf8b8ecad780bee6 2024-11-11T07:31:53,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T07:31:53,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:53,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:53,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:53,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:53,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:53,017 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/3af171c392424312b9068f947b72c132 is 50, key is test_row_0/C:col10/1731310310851/Put/seqid=0 2024-11-11T07:31:53,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742509_1685 (size=12101) 2024-11-11T07:31:53,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/3af171c392424312b9068f947b72c132 2024-11-11T07:31:53,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/73f342402564428c84a9a0f2aea8995c as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/73f342402564428c84a9a0f2aea8995c 2024-11-11T07:31:53,058 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/73f342402564428c84a9a0f2aea8995c, entries=200, sequenceid=135, filesize=38.8 K 2024-11-11T07:31:53,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/01543f79bca6463fbf8b8ecad780bee6 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/01543f79bca6463fbf8b8ecad780bee6 2024-11-11T07:31:53,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/01543f79bca6463fbf8b8ecad780bee6, entries=150, sequenceid=135, filesize=11.8 K 2024-11-11T07:31:53,066 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/3af171c392424312b9068f947b72c132 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/3af171c392424312b9068f947b72c132 2024-11-11T07:31:53,070 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/3af171c392424312b9068f947b72c132, entries=150, sequenceid=135, filesize=11.8 K 2024-11-11T07:31:53,071 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 298c53e334bb69b7b36af68612f3d3c8 in 1054ms, sequenceid=135, compaction requested=true 2024-11-11T07:31:53,071 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:53,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:53,071 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:53,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:53,071 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:53,072 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:53,072 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/B is initiating minor compaction (all files) 2024-11-11T07:31:53,072 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/B in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:53,072 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/e06d60a9169e4e08a8ab2df24fa63f3b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/92b9f5d41bd2466583b12e77a28af41e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/01543f79bca6463fbf8b8ecad780bee6] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=35.5 K 2024-11-11T07:31:53,072 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101815 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:53,072 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/A is initiating minor compaction (all files) 2024-11-11T07:31:53,073 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/A in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:53,073 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/c559194535114d029ac7b56757081cfd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/c71e3465912b44c887587fc266ab8897, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/73f342402564428c84a9a0f2aea8995c] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=99.4 K 2024-11-11T07:31:53,073 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:53,073 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/c559194535114d029ac7b56757081cfd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/c71e3465912b44c887587fc266ab8897, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/73f342402564428c84a9a0f2aea8995c] 2024-11-11T07:31:53,074 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting e06d60a9169e4e08a8ab2df24fa63f3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731310309527 2024-11-11T07:31:53,074 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting c559194535114d029ac7b56757081cfd, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731310309527 2024-11-11T07:31:53,075 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting c71e3465912b44c887587fc266ab8897, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731310309685 2024-11-11T07:31:53,075 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 92b9f5d41bd2466583b12e77a28af41e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731310309685 2024-11-11T07:31:53,075 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73f342402564428c84a9a0f2aea8995c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731310310851 2024-11-11T07:31:53,075 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 01543f79bca6463fbf8b8ecad780bee6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731310310851 2024-11-11T07:31:53,084 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#B#compaction#587 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:53,085 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/087414cb891846ee9aaf1eadb6fc926a is 50, key is test_row_0/B:col10/1731310310851/Put/seqid=0 2024-11-11T07:31:53,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:53,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:53,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:53,085 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:53,091 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:53,105 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111178b0adde7ff34cb0b3741704a704e2bf_298c53e334bb69b7b36af68612f3d3c8 store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:53,107 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111178b0adde7ff34cb0b3741704a704e2bf_298c53e334bb69b7b36af68612f3d3c8, store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:53,107 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111178b0adde7ff34cb0b3741704a704e2bf_298c53e334bb69b7b36af68612f3d3c8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:53,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742510_1686 (size=12409) 2024-11-11T07:31:53,124 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/087414cb891846ee9aaf1eadb6fc926a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/087414cb891846ee9aaf1eadb6fc926a 2024-11-11T07:31:53,128 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/B of 298c53e334bb69b7b36af68612f3d3c8 into 087414cb891846ee9aaf1eadb6fc926a(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:53,128 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:53,128 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/B, priority=13, startTime=1731310313071; duration=0sec 2024-11-11T07:31:53,128 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:53,128 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:B 2024-11-11T07:31:53,128 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:53,135 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48242 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:53,135 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/C is initiating minor compaction (all files) 2024-11-11T07:31:53,135 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/C in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:53,135 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/e50386f66aff4a868b9a06921d2df151, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/994d488d826741dcaabdac9db56047d2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/b238065298e644e0bbac8a6033ad1947, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/3af171c392424312b9068f947b72c132] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=47.1 K 2024-11-11T07:31:53,135 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting e50386f66aff4a868b9a06921d2df151, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=83, earliestPutTs=1731310308361 2024-11-11T07:31:53,136 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 994d488d826741dcaabdac9db56047d2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=96, earliestPutTs=1731310309527 2024-11-11T07:31:53,136 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting b238065298e644e0bbac8a6033ad1947, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731310309685 2024-11-11T07:31:53,136 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 3af171c392424312b9068f947b72c132, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731310310851 2024-11-11T07:31:53,145 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#C#compaction#589 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:53,145 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/a4fa9388740f42c9a889e9796cd8686b is 50, key is test_row_0/C:col10/1731310310851/Put/seqid=0 2024-11-11T07:31:53,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742511_1687 (size=4469) 2024-11-11T07:31:53,159 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#A#compaction#588 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:53,160 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/58ee058c7e044058992790a0c370390e is 175, key is test_row_0/A:col10/1731310310851/Put/seqid=0 2024-11-11T07:31:53,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T07:31:53,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-11T07:31:53,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:53,167 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T07:31:53,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:31:53,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:53,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:31:53,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:53,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:31:53,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:53,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742512_1688 (size=12375) 2024-11-11T07:31:53,197 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/a4fa9388740f42c9a889e9796cd8686b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/a4fa9388740f42c9a889e9796cd8686b 2024-11-11T07:31:53,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742513_1689 (size=31363) 2024-11-11T07:31:53,203 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/C of 298c53e334bb69b7b36af68612f3d3c8 into a4fa9388740f42c9a889e9796cd8686b(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:53,203 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:53,203 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/C, priority=12, startTime=1731310313085; duration=0sec 2024-11-11T07:31:53,203 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:53,203 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:C 2024-11-11T07:31:53,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:53,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:53,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fbdffd5b6910472484fafb4d797fe6e8_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310312051/Put/seqid=0 2024-11-11T07:31:53,217 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/58ee058c7e044058992790a0c370390e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/58ee058c7e044058992790a0c370390e 2024-11-11T07:31:53,222 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/A of 298c53e334bb69b7b36af68612f3d3c8 into 58ee058c7e044058992790a0c370390e(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:53,222 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:53,223 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/A, priority=13, startTime=1731310313071; duration=0sec 2024-11-11T07:31:53,223 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:53,223 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:A 2024-11-11T07:31:53,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742514_1690 (size=12304) 2024-11-11T07:31:53,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:53,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310373239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:53,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:53,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310373239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:53,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:53,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310373239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:53,249 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fbdffd5b6910472484fafb4d797fe6e8_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fbdffd5b6910472484fafb4d797fe6e8_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:53,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/a325a9b561284378b796fca869cfc474, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:53,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/a325a9b561284378b796fca869cfc474 is 175, key is test_row_0/A:col10/1731310312051/Put/seqid=0 2024-11-11T07:31:53,250 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,251 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:53,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742515_1691 (size=31105) 2024-11-11T07:31:53,291 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/a325a9b561284378b796fca869cfc474 2024-11-11T07:31:53,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/4006e153e6fd4ef0926faa0f07064d86 is 50, key is test_row_0/B:col10/1731310312051/Put/seqid=0 2024-11-11T07:31:53,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742516_1692 (size=12151) 2024-11-11T07:31:53,334 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/4006e153e6fd4ef0926faa0f07064d86 2024-11-11T07:31:53,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/a1816f36c09a416c90d200fe8b0332b1 is 50, key is test_row_0/C:col10/1731310312051/Put/seqid=0 2024-11-11T07:31:53,358 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310373354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:53,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:53,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310373355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,359 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:53,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:53,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310373356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:53,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:53,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742517_1693 (size=12151) 2024-11-11T07:31:53,387 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/a1816f36c09a416c90d200fe8b0332b1 2024-11-11T07:31:53,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/a325a9b561284378b796fca869cfc474 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/a325a9b561284378b796fca869cfc474 2024-11-11T07:31:53,397 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/a325a9b561284378b796fca869cfc474, entries=150, sequenceid=153, filesize=30.4 K 2024-11-11T07:31:53,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/4006e153e6fd4ef0926faa0f07064d86 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/4006e153e6fd4ef0926faa0f07064d86 2024-11-11T07:31:53,405 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/4006e153e6fd4ef0926faa0f07064d86, entries=150, sequenceid=153, filesize=11.9 K 2024-11-11T07:31:53,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/a1816f36c09a416c90d200fe8b0332b1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/a1816f36c09a416c90d200fe8b0332b1 2024-11-11T07:31:53,412 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/a1816f36c09a416c90d200fe8b0332b1, entries=150, sequenceid=153, filesize=11.9 K 2024-11-11T07:31:53,413 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 298c53e334bb69b7b36af68612f3d3c8 in 246ms, sequenceid=153, compaction requested=false 2024-11-11T07:31:53,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:53,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:53,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-11T07:31:53,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-11T07:31:53,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 881 msec 2024-11-11T07:31:53,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-11T07:31:53,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 875 msec 2024-11-11T07:31:53,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:53,568 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-11T07:31:53,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:31:53,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:53,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:31:53,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:53,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:31:53,568 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:53,580 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b109a838f46348c28bebf6e020f16740_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310313566/Put/seqid=0 2024-11-11T07:31:53,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742518_1694 (size=12304) 2024-11-11T07:31:53,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:53,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310373636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:53,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:53,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310373637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:53,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,643 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:53,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,645 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:53,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:53,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310373645, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:53,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:53,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-11T07:31:53,674 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:53,675 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:53,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees 2024-11-11T07:31:53,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T07:31:53,679 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:53,680 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=168, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:53,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:53,748 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:53,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310373745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:53,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:53,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310373755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:53,759 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:53,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310373755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,759 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:53,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,760 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:53,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T07:31:53,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-11T07:31:53,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:53,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:53,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:53,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:53,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:53,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:53,958 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:53,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310373955, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:53,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310373964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:53,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310373965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:53,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:53,969 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:53,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-11T07:31:53,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:53,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:53,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:53,990 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:53,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:53,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:53,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T07:31:53,995 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:54,004 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b109a838f46348c28bebf6e020f16740_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b109a838f46348c28bebf6e020f16740_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:54,007 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/24b077aaaa5b41559b4fa591c9f42dad, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:54,007 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/24b077aaaa5b41559b4fa591c9f42dad is 175, key is test_row_0/A:col10/1731310313566/Put/seqid=0 2024-11-11T07:31:54,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742519_1695 (size=31105) 2024-11-11T07:31:54,144 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-11T07:31:54,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:54,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:54,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:54,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,278 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310374275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:54,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:54,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:54,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310374275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:54,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:54,279 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:54,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:54,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:54,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:54,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310374275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:54,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:54,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-11T07:31:54,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:54,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:54,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:54,297 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T07:31:54,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-11T07:31:54,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:54,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:54,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:54,451 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,477 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:54,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54550 deadline: 1731310374474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:54,477 WARN [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_1' in 298c53e334bb69b7b36af68612f3d3c8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8196 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:31:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:54,478 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:54,495 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/24b077aaaa5b41559b4fa591c9f42dad 2024-11-11T07:31:54,524 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/133e9af7e71f45e291ba5ca7db4b3821 is 50, key is test_row_0/B:col10/1731310313566/Put/seqid=0 2024-11-11T07:31:54,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54518 deadline: 1731310374535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:54,541 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to ce2cf0244921,33345,1731310155369 for 'test_row_1' in 298c53e334bb69b7b36af68612f3d3c8 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8258 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-11T07:31:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:54,542 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:54,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742520_1696 (size=12151) 2024-11-11T07:31:54,646 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-11T07:31:54,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:54,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:54,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:54,646 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310374785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:54,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310374785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:54,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310374785, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:54,789 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:54,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-11T07:31:54,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:54,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:54,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:54,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T07:31:54,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-11T07:31:54,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:54,954 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:54,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:54,955 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:54,977 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/133e9af7e71f45e291ba5ca7db4b3821 2024-11-11T07:31:54,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/3fea9af7ddfd4375975f1bcc9d883ada is 50, key is test_row_0/C:col10/1731310313566/Put/seqid=0 2024-11-11T07:31:55,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742521_1697 (size=12151) 2024-11-11T07:31:55,113 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-11T07:31:55,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:55,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:55,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:55,113 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:55,113 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:55,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:55,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-11T07:31:55,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:55,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:55,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:55,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:55,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:55,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:55,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-11T07:31:55,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:55,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:55,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:55,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] handler.RSProcedureHandler(58): pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:55,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=169 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=169 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:55,458 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/3fea9af7ddfd4375975f1bcc9d883ada 2024-11-11T07:31:55,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/24b077aaaa5b41559b4fa591c9f42dad as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/24b077aaaa5b41559b4fa591c9f42dad 2024-11-11T07:31:55,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/24b077aaaa5b41559b4fa591c9f42dad, entries=150, sequenceid=172, filesize=30.4 K 2024-11-11T07:31:55,527 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/133e9af7e71f45e291ba5ca7db4b3821 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/133e9af7e71f45e291ba5ca7db4b3821 2024-11-11T07:31:55,533 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/133e9af7e71f45e291ba5ca7db4b3821, entries=150, sequenceid=172, filesize=11.9 K 2024-11-11T07:31:55,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/3fea9af7ddfd4375975f1bcc9d883ada as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/3fea9af7ddfd4375975f1bcc9d883ada 2024-11-11T07:31:55,537 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/3fea9af7ddfd4375975f1bcc9d883ada, entries=150, sequenceid=172, filesize=11.9 K 2024-11-11T07:31:55,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 298c53e334bb69b7b36af68612f3d3c8 in 1970ms, sequenceid=172, compaction requested=true 2024-11-11T07:31:55,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:55,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:55,538 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:55,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:55,538 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:55,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:55,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:55,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:55,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:55,539 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93573 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:55,539 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/A is initiating minor compaction (all files) 2024-11-11T07:31:55,539 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/A in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:55,539 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/58ee058c7e044058992790a0c370390e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/a325a9b561284378b796fca869cfc474, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/24b077aaaa5b41559b4fa591c9f42dad] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=91.4 K 2024-11-11T07:31:55,539 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:55,539 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/58ee058c7e044058992790a0c370390e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/a325a9b561284378b796fca869cfc474, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/24b077aaaa5b41559b4fa591c9f42dad] 2024-11-11T07:31:55,540 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36711 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:55,540 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/B is initiating minor compaction (all files) 2024-11-11T07:31:55,540 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/B in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:55,540 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/087414cb891846ee9aaf1eadb6fc926a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/4006e153e6fd4ef0926faa0f07064d86, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/133e9af7e71f45e291ba5ca7db4b3821] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=35.9 K 2024-11-11T07:31:55,540 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58ee058c7e044058992790a0c370390e, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731310310851 2024-11-11T07:31:55,540 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 087414cb891846ee9aaf1eadb6fc926a, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731310310851 2024-11-11T07:31:55,540 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting a325a9b561284378b796fca869cfc474, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731310312040 2024-11-11T07:31:55,540 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 4006e153e6fd4ef0926faa0f07064d86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731310312040 2024-11-11T07:31:55,540 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24b077aaaa5b41559b4fa591c9f42dad, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731310313228 2024-11-11T07:31:55,540 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 133e9af7e71f45e291ba5ca7db4b3821, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731310313228 2024-11-11T07:31:55,560 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#B#compaction#596 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:55,561 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/990739c8ad6a4b19965c1b90f6d6693d is 50, key is test_row_0/B:col10/1731310313566/Put/seqid=0 2024-11-11T07:31:55,562 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:55,569 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111176cea377b3454cd0b76f8be4d29eb0cc_298c53e334bb69b7b36af68612f3d3c8 store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:55,571 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111176cea377b3454cd0b76f8be4d29eb0cc_298c53e334bb69b7b36af68612f3d3c8, store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:55,571 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111176cea377b3454cd0b76f8be4d29eb0cc_298c53e334bb69b7b36af68612f3d3c8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:55,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742522_1698 (size=12561) 2024-11-11T07:31:55,583 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/990739c8ad6a4b19965c1b90f6d6693d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/990739c8ad6a4b19965c1b90f6d6693d 2024-11-11T07:31:55,598 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/B of 298c53e334bb69b7b36af68612f3d3c8 into 990739c8ad6a4b19965c1b90f6d6693d(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:55,598 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:55,598 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/B, priority=13, startTime=1731310315538; duration=0sec 2024-11-11T07:31:55,598 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:55,598 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:B 2024-11-11T07:31:55,598 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:55,599 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:55,599 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/C is initiating minor compaction (all files) 2024-11-11T07:31:55,599 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/C in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:55,599 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/a4fa9388740f42c9a889e9796cd8686b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/a1816f36c09a416c90d200fe8b0332b1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/3fea9af7ddfd4375975f1bcc9d883ada] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=35.8 K 2024-11-11T07:31:55,600 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting a4fa9388740f42c9a889e9796cd8686b, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731310310851 2024-11-11T07:31:55,601 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting a1816f36c09a416c90d200fe8b0332b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731310312040 2024-11-11T07:31:55,601 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fea9af7ddfd4375975f1bcc9d883ada, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731310313228 2024-11-11T07:31:55,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742523_1699 (size=4469) 2024-11-11T07:31:55,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=169 2024-11-11T07:31:55,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:55,603 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T07:31:55,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:31:55,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:55,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:31:55,603 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#A#compaction#597 average throughput is 0.60 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:55,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:55,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:31:55,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:55,604 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/b80270a69adf49c5b42af7ba09fbc8e1 is 175, key is test_row_0/A:col10/1731310313566/Put/seqid=0 2024-11-11T07:31:55,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bcdb64b504a740c6b92c6bd5f4f2b42a_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310313591/Put/seqid=0 2024-11-11T07:31:55,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742524_1700 (size=31515) 2024-11-11T07:31:55,637 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#C#compaction#599 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:55,637 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/32c9a0bf7c2041e1a12b66410ebde065 is 50, key is test_row_0/C:col10/1731310313566/Put/seqid=0 2024-11-11T07:31:55,646 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/b80270a69adf49c5b42af7ba09fbc8e1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/b80270a69adf49c5b42af7ba09fbc8e1 2024-11-11T07:31:55,659 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/A of 298c53e334bb69b7b36af68612f3d3c8 into b80270a69adf49c5b42af7ba09fbc8e1(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:55,659 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:55,659 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/A, priority=13, startTime=1731310315538; duration=0sec 2024-11-11T07:31:55,659 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:55,659 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:A 2024-11-11T07:31:55,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742525_1701 (size=12304) 2024-11-11T07:31:55,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742526_1702 (size=12527) 2024-11-11T07:31:55,798 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:55,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:55,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T07:31:55,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310375835, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:55,839 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:55,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310375836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:55,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:55,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310375838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:55,841 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:55,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:55,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:55,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310375947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:55,951 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:55,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:55,952 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:55,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310375947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:55,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:55,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:55,953 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:55,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310375947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:55,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:55,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:55,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:56,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:56,077 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bcdb64b504a740c6b92c6bd5f4f2b42a_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bcdb64b504a740c6b92c6bd5f4f2b42a_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:56,079 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/32c9a0bf7c2041e1a12b66410ebde065 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/32c9a0bf7c2041e1a12b66410ebde065 2024-11-11T07:31:56,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/da9b7bdfc4f941e4a520e7b2184c91fe, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:56,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/da9b7bdfc4f941e4a520e7b2184c91fe is 175, key is test_row_0/A:col10/1731310313591/Put/seqid=0 2024-11-11T07:31:56,089 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/C of 298c53e334bb69b7b36af68612f3d3c8 into 32c9a0bf7c2041e1a12b66410ebde065(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:56,089 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:56,089 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/C, priority=13, startTime=1731310315539; duration=0sec 2024-11-11T07:31:56,090 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:56,090 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:C 2024-11-11T07:31:56,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742527_1703 (size=31105) 2024-11-11T07:31:56,112 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=188, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/da9b7bdfc4f941e4a520e7b2184c91fe 2024-11-11T07:31:56,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/7e9570293ee546b3bff7f7d7458f1b33 is 50, key is test_row_0/B:col10/1731310313591/Put/seqid=0 2024-11-11T07:31:56,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742528_1704 (size=12151) 2024-11-11T07:31:56,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:56,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310376155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:56,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,157 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:56,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:56,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310376155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,158 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:56,158 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/7e9570293ee546b3bff7f7d7458f1b33 2024-11-11T07:31:56,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:56,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310376165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:56,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/b6408bd5604a459ab442139768c3ee3e is 50, key is test_row_0/C:col10/1731310313591/Put/seqid=0 2024-11-11T07:31:56,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742529_1705 (size=12151) 2024-11-11T07:31:56,216 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=188 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/b6408bd5604a459ab442139768c3ee3e 2024-11-11T07:31:56,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/da9b7bdfc4f941e4a520e7b2184c91fe as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/da9b7bdfc4f941e4a520e7b2184c91fe 2024-11-11T07:31:56,249 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/da9b7bdfc4f941e4a520e7b2184c91fe, entries=150, sequenceid=188, filesize=30.4 K 2024-11-11T07:31:56,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/7e9570293ee546b3bff7f7d7458f1b33 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/7e9570293ee546b3bff7f7d7458f1b33 2024-11-11T07:31:56,263 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/7e9570293ee546b3bff7f7d7458f1b33, entries=150, sequenceid=188, filesize=11.9 K 2024-11-11T07:31:56,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/b6408bd5604a459ab442139768c3ee3e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/b6408bd5604a459ab442139768c3ee3e 2024-11-11T07:31:56,273 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/b6408bd5604a459ab442139768c3ee3e, entries=150, sequenceid=188, filesize=11.9 K 2024-11-11T07:31:56,274 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 298c53e334bb69b7b36af68612f3d3c8 in 671ms, sequenceid=188, compaction requested=false 2024-11-11T07:31:56,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:56,274 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:56,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=169}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=169 2024-11-11T07:31:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=169 2024-11-11T07:31:56,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-11T07:31:56,294 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6020 sec 2024-11-11T07:31:56,296 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=168, table=TestAcidGuarantees in 2.6200 sec 2024-11-11T07:31:56,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:56,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-11T07:31:56,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:31:56,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:56,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:31:56,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:56,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:31:56,466 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:56,477 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fcb47b7d56064ec38aefa27acdcf0849_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310316465/Put/seqid=0 2024-11-11T07:31:56,496 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310376491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,496 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:56,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310376493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:56,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:56,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:56,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310376495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:56,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,500 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:56,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742530_1706 (size=14794) 2024-11-11T07:31:56,510 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:56,515 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111fcb47b7d56064ec38aefa27acdcf0849_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fcb47b7d56064ec38aefa27acdcf0849_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:56,516 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/8a508b02056e4ceba4e0b3fbe42cd93a, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:56,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/8a508b02056e4ceba4e0b3fbe42cd93a is 175, key is test_row_0/A:col10/1731310316465/Put/seqid=0 2024-11-11T07:31:56,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742531_1707 (size=39749) 2024-11-11T07:31:56,559 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=208, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/8a508b02056e4ceba4e0b3fbe42cd93a 2024-11-11T07:31:56,572 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/765bb4b958f24fbe8ed673cead4d18c2 is 50, key is test_row_0/B:col10/1731310316465/Put/seqid=0 2024-11-11T07:31:56,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310376605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:56,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310376605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:56,607 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310376605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:56,607 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,608 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:56,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742532_1708 (size=12151) 2024-11-11T07:31:56,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310376815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:56,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:56,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310376815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:56,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:56,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310376815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:56,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:57,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/765bb4b958f24fbe8ed673cead4d18c2 2024-11-11T07:31:57,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/49171b94b92843769787d77bbc683e0a is 50, key is test_row_0/C:col10/1731310316465/Put/seqid=0 2024-11-11T07:31:57,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742533_1709 (size=12151) 2024-11-11T07:31:57,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310377124, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:57,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:57,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310377125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:57,127 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310377125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:57,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/49171b94b92843769787d77bbc683e0a 2024-11-11T07:31:57,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/8a508b02056e4ceba4e0b3fbe42cd93a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/8a508b02056e4ceba4e0b3fbe42cd93a 2024-11-11T07:31:57,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/8a508b02056e4ceba4e0b3fbe42cd93a, entries=200, sequenceid=208, filesize=38.8 K 2024-11-11T07:31:57,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/765bb4b958f24fbe8ed673cead4d18c2 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/765bb4b958f24fbe8ed673cead4d18c2 2024-11-11T07:31:57,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/765bb4b958f24fbe8ed673cead4d18c2, entries=150, sequenceid=208, filesize=11.9 K 2024-11-11T07:31:57,485 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/49171b94b92843769787d77bbc683e0a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/49171b94b92843769787d77bbc683e0a 2024-11-11T07:31:57,488 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/49171b94b92843769787d77bbc683e0a, entries=150, sequenceid=208, filesize=11.9 K 2024-11-11T07:31:57,489 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 298c53e334bb69b7b36af68612f3d3c8 in 1023ms, sequenceid=208, compaction requested=true 2024-11-11T07:31:57,489 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:57,489 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:57,490 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:57,490 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/A is initiating minor compaction (all files) 2024-11-11T07:31:57,490 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/A in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:57,490 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/b80270a69adf49c5b42af7ba09fbc8e1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/da9b7bdfc4f941e4a520e7b2184c91fe, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/8a508b02056e4ceba4e0b3fbe42cd93a] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=100.0 K 2024-11-11T07:31:57,491 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:57,491 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/b80270a69adf49c5b42af7ba09fbc8e1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/da9b7bdfc4f941e4a520e7b2184c91fe, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/8a508b02056e4ceba4e0b3fbe42cd93a] 2024-11-11T07:31:57,491 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting b80270a69adf49c5b42af7ba09fbc8e1, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731310313228 2024-11-11T07:31:57,491 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting da9b7bdfc4f941e4a520e7b2184c91fe, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731310313591 2024-11-11T07:31:57,491 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8a508b02056e4ceba4e0b3fbe42cd93a, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731310315832 2024-11-11T07:31:57,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:57,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:57,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:57,506 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:57,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:57,507 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:57,507 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:57,507 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:57,507 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/B is initiating minor compaction (all files) 2024-11-11T07:31:57,507 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/B in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:57,508 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/990739c8ad6a4b19965c1b90f6d6693d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/7e9570293ee546b3bff7f7d7458f1b33, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/765bb4b958f24fbe8ed673cead4d18c2] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=36.0 K 2024-11-11T07:31:57,508 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 990739c8ad6a4b19965c1b90f6d6693d, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731310313228 2024-11-11T07:31:57,508 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e9570293ee546b3bff7f7d7458f1b33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731310313591 2024-11-11T07:31:57,508 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 765bb4b958f24fbe8ed673cead4d18c2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731310315832 2024-11-11T07:31:57,524 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:57,529 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#B#compaction#606 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:57,529 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/948590fb4b4d43baaff061f4581a958f is 50, key is test_row_0/B:col10/1731310316465/Put/seqid=0 2024-11-11T07:31:57,547 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411112c6f7963ebbb44f8919f844e706560c5_298c53e334bb69b7b36af68612f3d3c8 store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:57,549 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411112c6f7963ebbb44f8919f844e706560c5_298c53e334bb69b7b36af68612f3d3c8, store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:57,549 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411112c6f7963ebbb44f8919f844e706560c5_298c53e334bb69b7b36af68612f3d3c8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:57,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742535_1711 (size=4469) 2024-11-11T07:31:57,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742534_1710 (size=12663) 2024-11-11T07:31:57,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:57,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T07:31:57,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:31:57,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:57,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:31:57,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:57,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:31:57,643 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:57,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f6df429a13ed4280bf1defe2e828e96d_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310317638/Put/seqid=0 2024-11-11T07:31:57,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:57,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310377674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,679 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:57,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:57,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310377676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:57,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:57,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:57,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310377679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,686 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:57,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742536_1712 (size=17284) 2024-11-11T07:31:57,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:57,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310377784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,785 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:57,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310377794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,796 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310377794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:57,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-11T07:31:57,855 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:57,856 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:57,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees 2024-11-11T07:31:57,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-11T07:31:57,863 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:57,864 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=170, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:57,864 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:57,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-11T07:31:57,994 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#A#compaction#605 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:57,994 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/fa53da60d98b49c9a7b35ed70af4b08f is 175, key is test_row_0/A:col10/1731310316465/Put/seqid=0 2024-11-11T07:31:57,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:57,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310377995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:57,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:58,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:58,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310378004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:58,006 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:58,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310378005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:58,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742537_1713 (size=31617) 2024-11-11T07:31:58,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-11T07:31:58,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:58,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:58,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:58,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:58,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:58,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:58,021 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/948590fb4b4d43baaff061f4581a958f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/948590fb4b4d43baaff061f4581a958f 2024-11-11T07:31:58,023 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/fa53da60d98b49c9a7b35ed70af4b08f as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/fa53da60d98b49c9a7b35ed70af4b08f 2024-11-11T07:31:58,029 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/B of 298c53e334bb69b7b36af68612f3d3c8 into 948590fb4b4d43baaff061f4581a958f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:58,029 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:58,029 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/B, priority=13, startTime=1731310317506; duration=0sec 2024-11-11T07:31:58,029 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:58,029 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:B 2024-11-11T07:31:58,029 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:31:58,032 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/A of 298c53e334bb69b7b36af68612f3d3c8 into fa53da60d98b49c9a7b35ed70af4b08f(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:58,032 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:58,032 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/A, priority=13, startTime=1731310317489; duration=0sec 2024-11-11T07:31:58,032 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:58,032 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:A 2024-11-11T07:31:58,032 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:31:58,032 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/C is initiating minor compaction (all files) 2024-11-11T07:31:58,032 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/C in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:58,032 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/32c9a0bf7c2041e1a12b66410ebde065, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/b6408bd5604a459ab442139768c3ee3e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/49171b94b92843769787d77bbc683e0a] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=36.0 K 2024-11-11T07:31:58,032 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 32c9a0bf7c2041e1a12b66410ebde065, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1731310313228 2024-11-11T07:31:58,033 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting b6408bd5604a459ab442139768c3ee3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=188, earliestPutTs=1731310313591 2024-11-11T07:31:58,034 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 49171b94b92843769787d77bbc683e0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731310315832 2024-11-11T07:31:58,042 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#C#compaction#608 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:58,043 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/02fdfc3654ce42c0ba136d3f2fa7fd00 is 50, key is test_row_0/C:col10/1731310316465/Put/seqid=0 2024-11-11T07:31:58,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742538_1714 (size=12629) 2024-11-11T07:31:58,089 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,089 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/02fdfc3654ce42c0ba136d3f2fa7fd00 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/02fdfc3654ce42c0ba136d3f2fa7fd00 2024-11-11T07:31:58,098 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f6df429a13ed4280bf1defe2e828e96d_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f6df429a13ed4280bf1defe2e828e96d_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:58,099 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/C of 298c53e334bb69b7b36af68612f3d3c8 into 02fdfc3654ce42c0ba136d3f2fa7fd00(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:58,099 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:58,099 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/C, priority=13, startTime=1731310317506; duration=0sec 2024-11-11T07:31:58,099 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:58,099 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:C 2024-11-11T07:31:58,100 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/ca12b898149746aaa058edc081d25fe1, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:58,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/ca12b898149746aaa058edc081d25fe1 is 175, key is test_row_0/A:col10/1731310317638/Put/seqid=0 2024-11-11T07:31:58,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742539_1715 (size=48389) 2024-11-11T07:31:58,136 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=225, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/ca12b898149746aaa058edc081d25fe1 2024-11-11T07:31:58,157 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/9b528c2aabb24c58b7a2d3f8e7b91fc0 is 50, key is test_row_0/B:col10/1731310317638/Put/seqid=0 2024-11-11T07:31:58,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-11T07:31:58,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:58,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:58,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:58,175 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:58,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:58,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-11T07:31:58,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742540_1716 (size=12151) 2024-11-11T07:31:58,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/9b528c2aabb24c58b7a2d3f8e7b91fc0 2024-11-11T07:31:58,240 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/78951bfd7b7249b49864929ce8d8eb63 is 50, key is test_row_0/C:col10/1731310317638/Put/seqid=0 2024-11-11T07:31:58,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742541_1717 (size=12151) 2024-11-11T07:31:58,304 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=225 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/78951bfd7b7249b49864929ce8d8eb63 2024-11-11T07:31:58,306 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:58,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310378304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:58,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:58,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:58,308 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:58,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:58,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:58,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310378315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310378315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:58,323 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:58,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/ca12b898149746aaa058edc081d25fe1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/ca12b898149746aaa058edc081d25fe1 2024-11-11T07:31:58,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-11T07:31:58,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:58,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:58,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:58,334 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] handler.RSProcedureHandler(58): pid=171 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:58,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=171 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:58,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=171 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:58,343 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/ca12b898149746aaa058edc081d25fe1, entries=250, sequenceid=225, filesize=47.3 K 2024-11-11T07:31:58,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/9b528c2aabb24c58b7a2d3f8e7b91fc0 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/9b528c2aabb24c58b7a2d3f8e7b91fc0 2024-11-11T07:31:58,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/9b528c2aabb24c58b7a2d3f8e7b91fc0, entries=150, sequenceid=225, filesize=11.9 K 2024-11-11T07:31:58,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/78951bfd7b7249b49864929ce8d8eb63 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/78951bfd7b7249b49864929ce8d8eb63 2024-11-11T07:31:58,365 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/78951bfd7b7249b49864929ce8d8eb63, entries=150, sequenceid=225, filesize=11.9 K 2024-11-11T07:31:58,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 298c53e334bb69b7b36af68612f3d3c8 in 728ms, sequenceid=225, compaction requested=false 2024-11-11T07:31:58,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:58,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=171 2024-11-11T07:31:58,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:58,487 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-11T07:31:58,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:31:58,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:58,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:31:58,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:58,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:31:58,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:58,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-11T07:31:58,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119d09428884ea4289a8d40e69397ab028_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310317677/Put/seqid=0 2024-11-11T07:31:58,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742542_1718 (size=12304) 2024-11-11T07:31:58,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,537 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411119d09428884ea4289a8d40e69397ab028_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119d09428884ea4289a8d40e69397ab028_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:58,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/481a10558f6b4239aaf32d4b28b3d716, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:58,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/481a10558f6b4239aaf32d4b28b3d716 is 175, key is test_row_0/A:col10/1731310317677/Put/seqid=0 2024-11-11T07:31:58,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742543_1719 (size=31105) 2024-11-11T07:31:58,576 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=243, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/481a10558f6b4239aaf32d4b28b3d716 2024-11-11T07:31:58,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/f36e4e307fac47758b7dd253fe1ef03a is 50, key is test_row_0/B:col10/1731310317677/Put/seqid=0 2024-11-11T07:31:58,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742544_1720 (size=12151) 2024-11-11T07:31:58,624 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/f36e4e307fac47758b7dd253fe1ef03a 2024-11-11T07:31:58,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/fa44d48475ce4037aa07f571df8b00ec is 50, key is test_row_0/C:col10/1731310317677/Put/seqid=0 2024-11-11T07:31:58,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742545_1721 (size=12151) 2024-11-11T07:31:58,709 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=243 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/fa44d48475ce4037aa07f571df8b00ec 2024-11-11T07:31:58,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/481a10558f6b4239aaf32d4b28b3d716 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/481a10558f6b4239aaf32d4b28b3d716 2024-11-11T07:31:58,716 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/481a10558f6b4239aaf32d4b28b3d716, entries=150, sequenceid=243, filesize=30.4 K 2024-11-11T07:31:58,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/f36e4e307fac47758b7dd253fe1ef03a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/f36e4e307fac47758b7dd253fe1ef03a 2024-11-11T07:31:58,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,720 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/f36e4e307fac47758b7dd253fe1ef03a, entries=150, sequenceid=243, filesize=11.9 K 2024-11-11T07:31:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/fa44d48475ce4037aa07f571df8b00ec as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/fa44d48475ce4037aa07f571df8b00ec 2024-11-11T07:31:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,724 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/fa44d48475ce4037aa07f571df8b00ec, entries=150, sequenceid=243, filesize=11.9 K 2024-11-11T07:31:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,725 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=0 B/0 for 298c53e334bb69b7b36af68612f3d3c8 in 238ms, sequenceid=243, compaction requested=true 2024-11-11T07:31:58,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:58,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=171}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=171 2024-11-11T07:31:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=171 2024-11-11T07:31:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-11T07:31:58,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 863 msec 2024-11-11T07:31:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=170, table=TestAcidGuarantees in 873 msec 2024-11-11T07:31:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:58,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,915 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-11T07:31:58,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:31:58,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:58,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:31:58,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:58,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:31:58,916 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:31:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111124fd9da94d9c46fe8e4ab4aa074c855e_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310318910/Put/seqid=0 2024-11-11T07:31:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742546_1722 (size=17284) 2024-11-11T07:31:58,960 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:58,965 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111124fd9da94d9c46fe8e4ab4aa074c855e_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111124fd9da94d9c46fe8e4ab4aa074c855e_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:58,967 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/59c6d412d24948b6a67b814d3311f011, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:58,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/59c6d412d24948b6a67b814d3311f011 is 175, key is test_row_0/A:col10/1731310318910/Put/seqid=0 2024-11-11T07:31:58,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731310318974, max timestamp=1731310318989Num uniques 859; 2024-11-11T07:31:58,992 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-11T07:31:58,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310318974, max timestamp=1731310318989Num uniques 859; 2024-11-11T07:31:58,995 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:31:58,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731310318974, max timestamp=1731310318995Num uniques 859; 2024-11-11T07:31:58,998 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-11T07:31:59,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742547_1723 (size=48385) 2024-11-11T07:31:59,003 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/59c6d412d24948b6a67b814d3311f011 2024-11-11T07:31:59,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=170 2024-11-11T07:31:59,004 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:31:59,005 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:31:59,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees 2024-11-11T07:31:59,008 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:31:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T07:31:59,009 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=172, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:31:59,009 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=173, ppid=172, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:31:59,020 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/5c10e1f954334f4481f364931ee9db0e is 50, key is test_row_0/B:col10/1731310318910/Put/seqid=0 2024-11-11T07:31:59,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742548_1724 (size=9757) 2024-11-11T07:31:59,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731310318998, max timestamp=1731310319044Num uniques 1068; 2024-11-11T07:31:59,058 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=9978, suffix heap size=139040,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=218 2024-11-11T07:31:59,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731310318998, max timestamp=1731310319044Num uniques 1018; 2024-11-11T07:31:59,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T07:31:59,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731310318998, max timestamp=1731310319044Num uniques 1018; 2024-11-11T07:31:59,061 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-11T07:31:59,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T07:31:59,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:59,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310379131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:59,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:59,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310379131, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,139 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:59,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,140 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:59,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:59,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310379139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:59,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:59,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-11T07:31:59,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:59,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:59,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:59,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:59,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:59,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:59,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310379244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,247 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310379244, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:59,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310379255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,266 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:59,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-11T07:31:59,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:59,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:59,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:59,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:59,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:59,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T07:31:59,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/5c10e1f954334f4481f364931ee9db0e 2024-11-11T07:31:59,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:59,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310379454, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:59,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:59,460 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:59,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310379455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:59,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,463 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:59,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-11T07:31:59,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:59,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:59,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:59,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] handler.RSProcedureHandler(58): pid=173 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:59,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=173 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=173 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:31:59,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:59,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310379474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,478 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:59,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/6bb008c6b5174b41947225af326e3f50 is 50, key is test_row_0/C:col10/1731310318910/Put/seqid=0 2024-11-11T07:31:59,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742549_1725 (size=9757) 2024-11-11T07:31:59,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/6bb008c6b5174b41947225af326e3f50 2024-11-11T07:31:59,587 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/59c6d412d24948b6a67b814d3311f011 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/59c6d412d24948b6a67b814d3311f011 2024-11-11T07:31:59,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/59c6d412d24948b6a67b814d3311f011, entries=250, sequenceid=253, filesize=47.3 K 2024-11-11T07:31:59,620 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/5c10e1f954334f4481f364931ee9db0e as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/5c10e1f954334f4481f364931ee9db0e 2024-11-11T07:31:59,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/5c10e1f954334f4481f364931ee9db0e, entries=100, sequenceid=253, filesize=9.5 K 2024-11-11T07:31:59,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/6bb008c6b5174b41947225af326e3f50 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/6bb008c6b5174b41947225af326e3f50 2024-11-11T07:31:59,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/6bb008c6b5174b41947225af326e3f50, entries=100, sequenceid=253, filesize=9.5 K 2024-11-11T07:31:59,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 298c53e334bb69b7b36af68612f3d3c8 in 712ms, sequenceid=253, compaction requested=true 2024-11-11T07:31:59,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:59,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:31:59,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:59,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:31:59,628 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:59,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:59,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:31:59,628 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-11T07:31:59,628 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:59,629 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 159496 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:59,629 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46722 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:59,629 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/A is initiating minor compaction (all files) 2024-11-11T07:31:59,629 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/B is initiating minor compaction (all files) 2024-11-11T07:31:59,629 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/B in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:59,629 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/A in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:59,629 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/fa53da60d98b49c9a7b35ed70af4b08f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/ca12b898149746aaa058edc081d25fe1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/481a10558f6b4239aaf32d4b28b3d716, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/59c6d412d24948b6a67b814d3311f011] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=155.8 K 2024-11-11T07:31:59,629 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/948590fb4b4d43baaff061f4581a958f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/9b528c2aabb24c58b7a2d3f8e7b91fc0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/f36e4e307fac47758b7dd253fe1ef03a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/5c10e1f954334f4481f364931ee9db0e] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=45.6 K 2024-11-11T07:31:59,629 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:59,629 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/fa53da60d98b49c9a7b35ed70af4b08f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/ca12b898149746aaa058edc081d25fe1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/481a10558f6b4239aaf32d4b28b3d716, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/59c6d412d24948b6a67b814d3311f011] 2024-11-11T07:31:59,629 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 948590fb4b4d43baaff061f4581a958f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731310315832 2024-11-11T07:31:59,629 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa53da60d98b49c9a7b35ed70af4b08f, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731310315832 2024-11-11T07:31:59,630 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b528c2aabb24c58b7a2d3f8e7b91fc0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731310316486 2024-11-11T07:31:59,630 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting f36e4e307fac47758b7dd253fe1ef03a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731310317651 2024-11-11T07:31:59,630 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca12b898149746aaa058edc081d25fe1, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731310316486 2024-11-11T07:31:59,630 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c10e1f954334f4481f364931ee9db0e, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731310318910 2024-11-11T07:31:59,630 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 481a10558f6b4239aaf32d4b28b3d716, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731310317651 2024-11-11T07:31:59,630 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 59c6d412d24948b6a67b814d3311f011, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731310318832 2024-11-11T07:31:59,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T07:31:59,639 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:59,648 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#B#compaction#618 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:59,649 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/02c9412504f4433995a7fb3b1aea3636 is 50, key is test_row_0/B:col10/1731310318910/Put/seqid=0 2024-11-11T07:31:59,651 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411117c7ca6095ae443e0b61ed9ca2fc29cc0_298c53e334bb69b7b36af68612f3d3c8 store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:59,654 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411117c7ca6095ae443e0b61ed9ca2fc29cc0_298c53e334bb69b7b36af68612f3d3c8, store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:59,654 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411117c7ca6095ae443e0b61ed9ca2fc29cc0_298c53e334bb69b7b36af68612f3d3c8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:59,657 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=173 2024-11-11T07:31:59,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:59,657 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=134.18 KB heapSize=379.45 KB 2024-11-11T07:31:59,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:31:59,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:59,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:31:59,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:59,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:31:59,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:31:59,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742550_1726 (size=12799) 2024-11-11T07:31:59,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111128288b6249dd4b41b9264f9669e2253b_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310319100/Put/seqid=0 2024-11-11T07:31:59,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742551_1727 (size=4469) 2024-11-11T07:31:59,695 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/02c9412504f4433995a7fb3b1aea3636 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/02c9412504f4433995a7fb3b1aea3636 2024-11-11T07:31:59,697 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#A#compaction#617 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:59,697 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/0780849a28b5407e9fcdc001e387eb77 is 175, key is test_row_0/A:col10/1731310318910/Put/seqid=0 2024-11-11T07:31:59,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742552_1728 (size=12454) 2024-11-11T07:31:59,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:31:59,701 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/B of 298c53e334bb69b7b36af68612f3d3c8 into 02c9412504f4433995a7fb3b1aea3636(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:59,701 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:59,701 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/B, priority=12, startTime=1731310319628; duration=0sec 2024-11-11T07:31:59,701 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:31:59,701 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:B 2024-11-11T07:31:59,701 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:31:59,707 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46688 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:31:59,707 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/C is initiating minor compaction (all files) 2024-11-11T07:31:59,707 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/C in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:31:59,707 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/02fdfc3654ce42c0ba136d3f2fa7fd00, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/78951bfd7b7249b49864929ce8d8eb63, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/fa44d48475ce4037aa07f571df8b00ec, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/6bb008c6b5174b41947225af326e3f50] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=45.6 K 2024-11-11T07:31:59,707 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 02fdfc3654ce42c0ba136d3f2fa7fd00, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1731310315832 2024-11-11T07:31:59,708 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 78951bfd7b7249b49864929ce8d8eb63, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=225, earliestPutTs=1731310316486 2024-11-11T07:31:59,708 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting fa44d48475ce4037aa07f571df8b00ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=243, earliestPutTs=1731310317651 2024-11-11T07:31:59,708 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bb008c6b5174b41947225af326e3f50, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731310318910 2024-11-11T07:31:59,709 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111128288b6249dd4b41b9264f9669e2253b_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111128288b6249dd4b41b9264f9669e2253b_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:59,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/e08a465b869449318446afd6ac1acf15, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:31:59,711 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/e08a465b869449318446afd6ac1acf15 is 175, key is test_row_0/A:col10/1731310319100/Put/seqid=0 2024-11-11T07:31:59,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742554_1730 (size=31255) 2024-11-11T07:31:59,724 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/e08a465b869449318446afd6ac1acf15 2024-11-11T07:31:59,729 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#C#compaction#620 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:31:59,729 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/6cbc96bdf64349ffb9f2a257d41ad538 is 50, key is test_row_0/C:col10/1731310318910/Put/seqid=0 2024-11-11T07:31:59,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742553_1729 (size=31860) 2024-11-11T07:31:59,740 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/0780849a28b5407e9fcdc001e387eb77 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/0780849a28b5407e9fcdc001e387eb77 2024-11-11T07:31:59,745 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/A of 298c53e334bb69b7b36af68612f3d3c8 into 0780849a28b5407e9fcdc001e387eb77(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:59,745 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:59,745 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/A, priority=12, startTime=1731310319628; duration=0sec 2024-11-11T07:31:59,745 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:59,745 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:A 2024-11-11T07:31:59,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/32d36182afc847a4b3498ae1d5bc71fe is 50, key is test_row_0/B:col10/1731310319100/Put/seqid=0 2024-11-11T07:31:59,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:31:59,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:31:59,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742555_1731 (size=12765) 2024-11-11T07:31:59,782 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/6cbc96bdf64349ffb9f2a257d41ad538 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/6cbc96bdf64349ffb9f2a257d41ad538 2024-11-11T07:31:59,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742556_1732 (size=12301) 2024-11-11T07:31:59,794 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/C of 298c53e334bb69b7b36af68612f3d3c8 into 6cbc96bdf64349ffb9f2a257d41ad538(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:31:59,794 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:31:59,794 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/C, priority=12, startTime=1731310319628; duration=0sec 2024-11-11T07:31:59,794 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:31:59,794 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:C 2024-11-11T07:31:59,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:59,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310379790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:59,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310379792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:59,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310379795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:59,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,803 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:59,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:59,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310379904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:59,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:59,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:59,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310379914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:59,919 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:31:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,920 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:31:59,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310379915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:31:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:31:59,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:00,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310380115, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:00,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:00,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310380125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:00,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:00,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310380125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:00,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T07:32:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,146 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,147 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:00,188 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/32d36182afc847a4b3498ae1d5bc71fe 2024-11-11T07:32:00,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/341a1df16d0d4d9e9f55b0381c9cbed9 is 50, key is test_row_0/C:col10/1731310319100/Put/seqid=0 2024-11-11T07:32:00,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742557_1733 (size=12301) 2024-11-11T07:32:00,242 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/341a1df16d0d4d9e9f55b0381c9cbed9 2024-11-11T07:32:00,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/e08a465b869449318446afd6ac1acf15 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/e08a465b869449318446afd6ac1acf15 2024-11-11T07:32:00,251 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/e08a465b869449318446afd6ac1acf15, entries=150, sequenceid=306, filesize=30.5 K 2024-11-11T07:32:00,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/32d36182afc847a4b3498ae1d5bc71fe as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/32d36182afc847a4b3498ae1d5bc71fe 2024-11-11T07:32:00,256 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/32d36182afc847a4b3498ae1d5bc71fe, entries=150, sequenceid=306, filesize=12.0 K 2024-11-11T07:32:00,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/341a1df16d0d4d9e9f55b0381c9cbed9 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/341a1df16d0d4d9e9f55b0381c9cbed9 2024-11-11T07:32:00,260 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/341a1df16d0d4d9e9f55b0381c9cbed9, entries=150, sequenceid=306, filesize=12.0 K 2024-11-11T07:32:00,261 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~379.41 KB/388512, currentSize=46.96 KB/48090 for 298c53e334bb69b7b36af68612f3d3c8 in 604ms, sequenceid=306, compaction requested=false 2024-11-11T07:32:00,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:32:00,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:00,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=173}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=173 2024-11-11T07:32:00,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=173 2024-11-11T07:32:00,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=173, resume processing ppid=172 2024-11-11T07:32:00,278 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=173, ppid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2580 sec 2024-11-11T07:32:00,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=172, table=TestAcidGuarantees in 1.2770 sec 2024-11-11T07:32:00,437 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-11T07:32:00,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:32:00,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:00,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:32:00,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:00,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:32:00,438 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:00,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b634774760f847719abd13ae8a5d7afa_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310319791/Put/seqid=0 2024-11-11T07:32:00,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742558_1734 (size=14994) 2024-11-11T07:32:00,467 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:32:00,470 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b634774760f847719abd13ae8a5d7afa_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b634774760f847719abd13ae8a5d7afa_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:00,471 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/6dc704c7618d4837b65cec53a77ed6db, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:32:00,471 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/6dc704c7618d4837b65cec53a77ed6db is 175, key is test_row_0/A:col10/1731310319791/Put/seqid=0 2024-11-11T07:32:00,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731310320455, max timestamp=1731310320489Num uniques 859; 2024-11-11T07:32:00,492 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-11T07:32:00,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742559_1735 (size=39949) 2024-11-11T07:32:00,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731310320455, max timestamp=1731310320489Num uniques 859; 2024-11-11T07:32:00,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731310320455, max timestamp=1731310320489Num uniques 859; 2024-11-11T07:32:00,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T07:32:00,497 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-11T07:32:00,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310380533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,545 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:00,550 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310380542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:00,550 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310380544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:00,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:00,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:00,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:00,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310380654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:00,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310380654, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,660 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:00,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310380655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:00,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:00,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310380865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,867 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:00,867 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:00,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310380865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:00,868 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:00,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310380866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:00,868 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:00,895 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=320, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/6dc704c7618d4837b65cec53a77ed6db 2024-11-11T07:32:00,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/a275288531534c29934df4deeff2ddcd is 50, key is test_row_0/B:col10/1731310319791/Put/seqid=0 2024-11-11T07:32:00,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742560_1736 (size=12301) 2024-11-11T07:32:00,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/a275288531534c29934df4deeff2ddcd 2024-11-11T07:32:00,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/bd70238f0994419eabcd10dbdb414bca is 50, key is test_row_0/C:col10/1731310319791/Put/seqid=0 2024-11-11T07:32:00,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742561_1737 (size=12301) 2024-11-11T07:32:00,975 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/bd70238f0994419eabcd10dbdb414bca 2024-11-11T07:32:00,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/6dc704c7618d4837b65cec53a77ed6db as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/6dc704c7618d4837b65cec53a77ed6db 2024-11-11T07:32:00,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/6dc704c7618d4837b65cec53a77ed6db, entries=200, sequenceid=320, filesize=39.0 K 2024-11-11T07:32:00,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/a275288531534c29934df4deeff2ddcd as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/a275288531534c29934df4deeff2ddcd 2024-11-11T07:32:00,995 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/a275288531534c29934df4deeff2ddcd, entries=150, sequenceid=320, filesize=12.0 K 2024-11-11T07:32:00,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/bd70238f0994419eabcd10dbdb414bca as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/bd70238f0994419eabcd10dbdb414bca 2024-11-11T07:32:00,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/bd70238f0994419eabcd10dbdb414bca, entries=150, sequenceid=320, filesize=12.0 K 2024-11-11T07:32:01,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=131.15 KB/134300 for 298c53e334bb69b7b36af68612f3d3c8 in 565ms, sequenceid=320, compaction requested=true 2024-11-11T07:32:01,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:32:01,002 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:32:01,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:32:01,003 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103064 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:32:01,003 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/A is initiating minor compaction (all files) 2024-11-11T07:32:01,003 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/A in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:01,003 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/0780849a28b5407e9fcdc001e387eb77, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/e08a465b869449318446afd6ac1acf15, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/6dc704c7618d4837b65cec53a77ed6db] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=100.6 K 2024-11-11T07:32:01,003 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:01,003 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/0780849a28b5407e9fcdc001e387eb77, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/e08a465b869449318446afd6ac1acf15, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/6dc704c7618d4837b65cec53a77ed6db] 2024-11-11T07:32:01,004 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0780849a28b5407e9fcdc001e387eb77, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731310317651 2024-11-11T07:32:01,004 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting e08a465b869449318446afd6ac1acf15, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731310319097 2024-11-11T07:32:01,004 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dc704c7618d4837b65cec53a77ed6db, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731310319785 2024-11-11T07:32:01,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:32:01,006 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:32:01,007 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:32:01,007 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/B is initiating minor compaction (all files) 2024-11-11T07:32:01,007 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/B in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:01,007 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/02c9412504f4433995a7fb3b1aea3636, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/32d36182afc847a4b3498ae1d5bc71fe, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/a275288531534c29934df4deeff2ddcd] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=36.5 K 2024-11-11T07:32:01,008 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 02c9412504f4433995a7fb3b1aea3636, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731310317651 2024-11-11T07:32:01,008 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 32d36182afc847a4b3498ae1d5bc71fe, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731310319097 2024-11-11T07:32:01,009 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting a275288531534c29934df4deeff2ddcd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731310319785 2024-11-11T07:32:01,011 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:32:01,014 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111a982b3cddae0434da83c9c6bd86bb108_298c53e334bb69b7b36af68612f3d3c8 store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:32:01,015 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:32:01,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:32:01,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:32:01,016 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:32:01,016 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111a982b3cddae0434da83c9c6bd86bb108_298c53e334bb69b7b36af68612f3d3c8, store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:32:01,016 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111a982b3cddae0434da83c9c6bd86bb108_298c53e334bb69b7b36af68612f3d3c8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:32:01,024 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#B#compaction#627 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:32:01,025 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/526c59f41a3f435ea1fe8270a310d02b is 50, key is test_row_0/B:col10/1731310319791/Put/seqid=0 2024-11-11T07:32:01,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742563_1739 (size=4469) 2024-11-11T07:32:01,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742562_1738 (size=13051) 2024-11-11T07:32:01,098 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/526c59f41a3f435ea1fe8270a310d02b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/526c59f41a3f435ea1fe8270a310d02b 2024-11-11T07:32:01,105 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/B of 298c53e334bb69b7b36af68612f3d3c8 into 526c59f41a3f435ea1fe8270a310d02b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:32:01,105 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:32:01,105 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/B, priority=13, startTime=1731310321006; duration=0sec 2024-11-11T07:32:01,105 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:32:01,105 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:B 2024-11-11T07:32:01,105 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-11T07:32:01,107 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37367 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-11T07:32:01,107 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/C is initiating minor compaction (all files) 2024-11-11T07:32:01,107 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/C in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:01,107 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/6cbc96bdf64349ffb9f2a257d41ad538, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/341a1df16d0d4d9e9f55b0381c9cbed9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/bd70238f0994419eabcd10dbdb414bca] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=36.5 K 2024-11-11T07:32:01,108 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cbc96bdf64349ffb9f2a257d41ad538, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1731310317651 2024-11-11T07:32:01,108 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 341a1df16d0d4d9e9f55b0381c9cbed9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731310319097 2024-11-11T07:32:01,109 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting bd70238f0994419eabcd10dbdb414bca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731310319785 2024-11-11T07:32:01,126 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#C#compaction#628 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:32:01,127 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/419b996554614bf2986e70a498dfe474 is 50, key is test_row_0/C:col10/1731310319791/Put/seqid=0 2024-11-11T07:32:01,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742564_1740 (size=13017) 2024-11-11T07:32:01,150 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/419b996554614bf2986e70a498dfe474 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/419b996554614bf2986e70a498dfe474 2024-11-11T07:32:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-11T07:32:01,154 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:32:01,156 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:32:01,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees 2024-11-11T07:32:01,158 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:32:01,159 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=174, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:32:01,159 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=175, ppid=174, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:32:01,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-11T07:32:01,160 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/C of 298c53e334bb69b7b36af68612f3d3c8 into 419b996554614bf2986e70a498dfe474(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:32:01,161 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:32:01,161 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/C, priority=13, startTime=1731310321016; duration=0sec 2024-11-11T07:32:01,161 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:32:01,161 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:C 2024-11-11T07:32:01,178 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=137.86 KB heapSize=393.78 KB 2024-11-11T07:32:01,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:01,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731310320455, max timestamp=1731310321177Num uniques 1009; 2024-11-11T07:32:01,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731310320455, max timestamp=1731310321177Num uniques 1118; 2024-11-11T07:32:01,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46213, new segment data size=9978, suffix heap size=132904,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=218 2024-11-11T07:32:01,180 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/ce2cf0244921:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=51198, new segment data size=9978, suffix heap size=144240,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1118, new segment cells count=218 2024-11-11T07:32:01,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:32:01,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:32:01,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:32:01,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-11T07:32:01,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:32:01,183 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-11T07:32:01,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110ca150a8332b436b94f13ded9d834b17_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310321177/Put/seqid=0 2024-11-11T07:32:01,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:01,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310381239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:01,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,248 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:01,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:01,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310381242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,252 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:01,259 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:01,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310381248, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,260 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:01,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742565_1741 (size=12454) 2024-11-11T07:32:01,264 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:32:01,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-11T07:32:01,267 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411110ca150a8332b436b94f13ded9d834b17_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110ca150a8332b436b94f13ded9d834b17_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:01,268 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/f508a24a98a241e3b0efef962d6b1705, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:32:01,268 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/f508a24a98a241e3b0efef962d6b1705 is 175, key is test_row_0/A:col10/1731310321177/Put/seqid=0 2024-11-11T07:32:01,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742566_1742 (size=31255) 2024-11-11T07:32:01,307 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/f508a24a98a241e3b0efef962d6b1705 2024-11-11T07:32:01,311 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-11T07:32:01,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:01,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:32:01,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:01,311 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:01,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:01,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:01,321 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/65911a4b448b4c0981b506e11b6f93c7 is 50, key is test_row_0/B:col10/1731310321177/Put/seqid=0 2024-11-11T07:32:01,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742567_1743 (size=12301) 2024-11-11T07:32:01,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:01,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310381359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:01,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:01,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310381359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:01,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:01,367 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:01,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310381365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:01,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,367 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:01,463 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#A#compaction#626 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:32:01,463 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/95127becff0b458ebfa8178d22b07150 is 175, key is test_row_0/A:col10/1731310319791/Put/seqid=0 2024-11-11T07:32:01,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-11T07:32:01,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:01,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:32:01,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:01,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:01,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:01,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:01,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-11T07:32:01,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742568_1744 (size=32005) 2024-11-11T07:32:01,497 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/95127becff0b458ebfa8178d22b07150 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/95127becff0b458ebfa8178d22b07150 2024-11-11T07:32:01,515 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/A of 298c53e334bb69b7b36af68612f3d3c8 into 95127becff0b458ebfa8178d22b07150(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:32:01,515 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:32:01,515 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/A, priority=13, startTime=1731310321002; duration=0sec 2024-11-11T07:32:01,515 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:32:01,515 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:A 2024-11-11T07:32:01,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:01,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310381574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:01,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310381574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:01,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:01,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310381575, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,578 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:01,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-11T07:32:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:32:01,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:01,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] handler.RSProcedureHandler(58): pid=175 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:01,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=175 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:01,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=175 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:01,733 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/65911a4b448b4c0981b506e11b6f93c7 2024-11-11T07:32:01,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/8d2f57b9e1404ef3ae48f3b2a42ac3ad is 50, key is test_row_0/C:col10/1731310321177/Put/seqid=0 2024-11-11T07:32:01,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742569_1745 (size=12301) 2024-11-11T07:32:01,760 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.57 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/8d2f57b9e1404ef3ae48f3b2a42ac3ad 2024-11-11T07:32:01,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/f508a24a98a241e3b0efef962d6b1705 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/f508a24a98a241e3b0efef962d6b1705 2024-11-11T07:32:01,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/f508a24a98a241e3b0efef962d6b1705, entries=150, sequenceid=360, filesize=30.5 K 2024-11-11T07:32:01,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/65911a4b448b4c0981b506e11b6f93c7 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/65911a4b448b4c0981b506e11b6f93c7 2024-11-11T07:32:01,776 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/65911a4b448b4c0981b506e11b6f93c7, entries=150, sequenceid=360, filesize=12.0 K 2024-11-11T07:32:01,776 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/8d2f57b9e1404ef3ae48f3b2a42ac3ad as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/8d2f57b9e1404ef3ae48f3b2a42ac3ad 2024-11-11T07:32:01,780 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/8d2f57b9e1404ef3ae48f3b2a42ac3ad, entries=150, sequenceid=360, filesize=12.0 K 2024-11-11T07:32:01,781 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.93 KB/70585, heapSize ~187.99 KB/192504, currentSize=114.05 KB/116790 for 298c53e334bb69b7b36af68612f3d3c8 in 603ms, sequenceid=360, compaction requested=false 2024-11-11T07:32:01,781 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:32:01,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=175 2024-11-11T07:32:01,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:01,783 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:32:01,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:32:01,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:01,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:32:01,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:01,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:32:01,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:01,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-11T07:32:01,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114718999e3ebf4ee3bf836daebfdd5b17_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310321237/Put/seqid=0 2024-11-11T07:32:01,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742570_1746 (size=12454) 2024-11-11T07:32:01,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:32:01,832 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411114718999e3ebf4ee3bf836daebfdd5b17_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114718999e3ebf4ee3bf836daebfdd5b17_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:01,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/07a17c28ce674a5ca8d1f64420f6963b, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:32:01,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/07a17c28ce674a5ca8d1f64420f6963b is 175, key is test_row_0/A:col10/1731310321237/Put/seqid=0 2024-11-11T07:32:01,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742571_1747 (size=31255) 2024-11-11T07:32:01,872 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=381, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/07a17c28ce674a5ca8d1f64420f6963b 2024-11-11T07:32:01,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:01,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:32:01,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/265549c9798e47f69496698316f0f397 is 50, key is test_row_0/B:col10/1731310321237/Put/seqid=0 2024-11-11T07:32:01,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:01,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310381911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,915 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:01,918 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:01,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310381912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,919 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:01,920 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:01,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310381915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:01,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:01,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:01,941 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742572_1748 (size=12301) 2024-11-11T07:32:01,941 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/265549c9798e47f69496698316f0f397 2024-11-11T07:32:01,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/c2a6177910d24e3ca47f3b85f76e8aca is 50, key is test_row_0/C:col10/1731310321237/Put/seqid=0 2024-11-11T07:32:01,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742573_1749 (size=12301) 2024-11-11T07:32:01,996 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/c2a6177910d24e3ca47f3b85f76e8aca 2024-11-11T07:32:02,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/07a17c28ce674a5ca8d1f64420f6963b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/07a17c28ce674a5ca8d1f64420f6963b 2024-11-11T07:32:02,005 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/07a17c28ce674a5ca8d1f64420f6963b, entries=150, sequenceid=381, filesize=30.5 K 2024-11-11T07:32:02,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/265549c9798e47f69496698316f0f397 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/265549c9798e47f69496698316f0f397 2024-11-11T07:32:02,010 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/265549c9798e47f69496698316f0f397, entries=150, sequenceid=381, filesize=12.0 K 2024-11-11T07:32:02,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/c2a6177910d24e3ca47f3b85f76e8aca as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/c2a6177910d24e3ca47f3b85f76e8aca 2024-11-11T07:32:02,016 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/c2a6177910d24e3ca47f3b85f76e8aca, entries=150, sequenceid=381, filesize=12.0 K 2024-11-11T07:32:02,017 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 298c53e334bb69b7b36af68612f3d3c8 in 234ms, sequenceid=381, compaction requested=true 2024-11-11T07:32:02,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:32:02,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:02,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=175}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=175 2024-11-11T07:32:02,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=175 2024-11-11T07:32:02,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=175, resume processing ppid=174 2024-11-11T07:32:02,024 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=175, ppid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 861 msec 2024-11-11T07:32:02,028 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=174, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=174, table=TestAcidGuarantees in 869 msec 2024-11-11T07:32:02,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:02,031 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T07:32:02,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:32:02,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:02,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:32:02,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:02,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:32:02,031 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:02,036 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b16b18cde6bd4f59aeedfced5be5c7c2_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310322029/Put/seqid=0 2024-11-11T07:32:02,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742574_1750 (size=14994) 2024-11-11T07:32:02,059 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:32:02,062 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b16b18cde6bd4f59aeedfced5be5c7c2_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b16b18cde6bd4f59aeedfced5be5c7c2_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:02,064 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/b6704bc8615544bdbc9ef274d84be5eb, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:32:02,064 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/b6704bc8615544bdbc9ef274d84be5eb is 175, key is test_row_0/A:col10/1731310322029/Put/seqid=0 2024-11-11T07:32:02,083 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310382079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:02,085 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310382080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:02,086 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310382080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,086 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:02,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742575_1751 (size=39949) 2024-11-11T07:32:02,096 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=396, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/b6704bc8615544bdbc9ef274d84be5eb 2024-11-11T07:32:02,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/cf00181e7c7342a18201b93176b25735 is 50, key is test_row_0/B:col10/1731310322029/Put/seqid=0 2024-11-11T07:32:02,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742576_1752 (size=12301) 2024-11-11T07:32:02,153 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/cf00181e7c7342a18201b93176b25735 2024-11-11T07:32:02,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/9be869f71b4c44cea5785cd6a1f72c7d is 50, key is test_row_0/C:col10/1731310322029/Put/seqid=0 2024-11-11T07:32:02,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310382195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:02,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,199 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:02,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310382195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,200 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:02,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310382195, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:02,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:02,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742577_1753 (size=12301) 2024-11-11T07:32:02,208 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=396 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/9be869f71b4c44cea5785cd6a1f72c7d 2024-11-11T07:32:02,212 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/b6704bc8615544bdbc9ef274d84be5eb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/b6704bc8615544bdbc9ef274d84be5eb 2024-11-11T07:32:02,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/b6704bc8615544bdbc9ef274d84be5eb, entries=200, sequenceid=396, filesize=39.0 K 2024-11-11T07:32:02,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/cf00181e7c7342a18201b93176b25735 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/cf00181e7c7342a18201b93176b25735 2024-11-11T07:32:02,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/cf00181e7c7342a18201b93176b25735, entries=150, sequenceid=396, filesize=12.0 K 2024-11-11T07:32:02,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/9be869f71b4c44cea5785cd6a1f72c7d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/9be869f71b4c44cea5785cd6a1f72c7d 2024-11-11T07:32:02,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/9be869f71b4c44cea5785cd6a1f72c7d, entries=150, sequenceid=396, filesize=12.0 K 2024-11-11T07:32:02,225 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 298c53e334bb69b7b36af68612f3d3c8 in 194ms, sequenceid=396, compaction requested=true 2024-11-11T07:32:02,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:32:02,226 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:32:02,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:A, priority=-2147483648, current under compaction store size is 1 2024-11-11T07:32:02,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:32:02,226 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:32:02,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:B, priority=-2147483648, current under compaction store size is 2 2024-11-11T07:32:02,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:32:02,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:C, priority=-2147483648, current under compaction store size is 3 2024-11-11T07:32:02,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:32:02,227 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:32:02,227 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/B is initiating minor compaction (all files) 2024-11-11T07:32:02,227 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/B in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:02,227 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/526c59f41a3f435ea1fe8270a310d02b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/65911a4b448b4c0981b506e11b6f93c7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/265549c9798e47f69496698316f0f397, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/cf00181e7c7342a18201b93176b25735] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=48.8 K 2024-11-11T07:32:02,227 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:32:02,227 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/A is initiating minor compaction (all files) 2024-11-11T07:32:02,227 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/A in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:02,227 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/95127becff0b458ebfa8178d22b07150, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/f508a24a98a241e3b0efef962d6b1705, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/07a17c28ce674a5ca8d1f64420f6963b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/b6704bc8615544bdbc9ef274d84be5eb] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=131.3 K 2024-11-11T07:32:02,227 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:02,227 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. files: [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/95127becff0b458ebfa8178d22b07150, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/f508a24a98a241e3b0efef962d6b1705, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/07a17c28ce674a5ca8d1f64420f6963b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/b6704bc8615544bdbc9ef274d84be5eb] 2024-11-11T07:32:02,228 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 526c59f41a3f435ea1fe8270a310d02b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731310319785 2024-11-11T07:32:02,228 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95127becff0b458ebfa8178d22b07150, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731310319785 2024-11-11T07:32:02,228 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 65911a4b448b4c0981b506e11b6f93c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731310320529 2024-11-11T07:32:02,228 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting f508a24a98a241e3b0efef962d6b1705, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731310320529 2024-11-11T07:32:02,228 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 265549c9798e47f69496698316f0f397, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731310321237 2024-11-11T07:32:02,229 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07a17c28ce674a5ca8d1f64420f6963b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731310321237 2024-11-11T07:32:02,229 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting cf00181e7c7342a18201b93176b25735, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731310321913 2024-11-11T07:32:02,229 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] compactions.Compactor(225): Compacting b6704bc8615544bdbc9ef274d84be5eb, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731310321896 2024-11-11T07:32:02,239 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:32:02,243 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#B#compaction#639 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:32:02,243 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/4ac3c92b7b7343988b02e5fcfac8d951 is 50, key is test_row_0/B:col10/1731310322029/Put/seqid=0 2024-11-11T07:32:02,253 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241111f332953d6dfe4a5da5978051a066ca21_298c53e334bb69b7b36af68612f3d3c8 store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:32:02,255 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241111f332953d6dfe4a5da5978051a066ca21_298c53e334bb69b7b36af68612f3d3c8, store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:32:02,256 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111f332953d6dfe4a5da5978051a066ca21_298c53e334bb69b7b36af68612f3d3c8 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:32:02,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742578_1754 (size=13187) 2024-11-11T07:32:02,281 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/4ac3c92b7b7343988b02e5fcfac8d951 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/4ac3c92b7b7343988b02e5fcfac8d951 2024-11-11T07:32:02,286 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/B of 298c53e334bb69b7b36af68612f3d3c8 into 4ac3c92b7b7343988b02e5fcfac8d951(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:32:02,286 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:32:02,286 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/B, priority=12, startTime=1731310322226; duration=0sec 2024-11-11T07:32:02,286 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-11T07:32:02,286 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:B 2024-11-11T07:32:02,286 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-11T07:32:02,287 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-11T07:32:02,287 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1541): 298c53e334bb69b7b36af68612f3d3c8/C is initiating minor compaction (all files) 2024-11-11T07:32:02,287 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 298c53e334bb69b7b36af68612f3d3c8/C in TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:02,287 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/419b996554614bf2986e70a498dfe474, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/8d2f57b9e1404ef3ae48f3b2a42ac3ad, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/c2a6177910d24e3ca47f3b85f76e8aca, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/9be869f71b4c44cea5785cd6a1f72c7d] into tmpdir=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp, totalSize=48.8 K 2024-11-11T07:32:02,287 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 419b996554614bf2986e70a498dfe474, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731310319785 2024-11-11T07:32:02,289 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d2f57b9e1404ef3ae48f3b2a42ac3ad, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=360, earliestPutTs=1731310320529 2024-11-11T07:32:02,289 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting c2a6177910d24e3ca47f3b85f76e8aca, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731310321237 2024-11-11T07:32:02,291 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] compactions.Compactor(225): Compacting 9be869f71b4c44cea5785cd6a1f72c7d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=396, earliestPutTs=1731310321913 2024-11-11T07:32:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-11-11T07:32:02,295 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:32:02,295 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-11T07:32:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees 2024-11-11T07:32:02,297 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-11T07:32:02,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742579_1755 (size=4469) 2024-11-11T07:32:02,299 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=176, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-11T07:32:02,299 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=177, ppid=176, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-11T07:32:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-11T07:32:02,304 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#A#compaction#638 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:32:02,305 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/340ebbd80ae043d6b7797b8635a14d8d is 175, key is test_row_0/A:col10/1731310322029/Put/seqid=0 2024-11-11T07:32:02,313 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 298c53e334bb69b7b36af68612f3d3c8#C#compaction#640 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-11T07:32:02,314 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/49a52d5e4cf24963bbd12afd55433bcb is 50, key is test_row_0/C:col10/1731310322029/Put/seqid=0 2024-11-11T07:32:02,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742580_1756 (size=32141) 2024-11-11T07:32:02,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742581_1757 (size=13153) 2024-11-11T07:32:02,348 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/49a52d5e4cf24963bbd12afd55433bcb as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/49a52d5e4cf24963bbd12afd55433bcb 2024-11-11T07:32:02,349 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/340ebbd80ae043d6b7797b8635a14d8d as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/340ebbd80ae043d6b7797b8635a14d8d 2024-11-11T07:32:02,357 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/A of 298c53e334bb69b7b36af68612f3d3c8 into 340ebbd80ae043d6b7797b8635a14d8d(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:32:02,357 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:32:02,357 INFO [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/A, priority=12, startTime=1731310322226; duration=0sec 2024-11-11T07:32:02,357 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:32:02,357 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:A 2024-11-11T07:32:02,357 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 298c53e334bb69b7b36af68612f3d3c8/C of 298c53e334bb69b7b36af68612f3d3c8 into 49a52d5e4cf24963bbd12afd55433bcb(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-11T07:32:02,357 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:32:02,358 INFO [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., storeName=298c53e334bb69b7b36af68612f3d3c8/C, priority=12, startTime=1731310322226; duration=0sec 2024-11-11T07:32:02,358 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:32:02,358 DEBUG [RS:0;ce2cf0244921:33345-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:C 2024-11-11T07:32:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-11T07:32:02,411 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:32:02,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:32:02,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:02,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:32:02,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:02,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:32:02,411 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:02,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111422318d85ad14bb9842391ef2a18bd7b_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310322078/Put/seqid=0 2024-11-11T07:32:02,444 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:02,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310382438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,445 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:02,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:02,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310382438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:02,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310382439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,446 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:02,452 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-11T07:32:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:32:02,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:02,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:02,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:02,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:02,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742582_1758 (size=12454) 2024-11-11T07:32:02,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:02,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310382554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:02,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:02,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310382554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:02,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,559 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:02,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:02,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310382555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:02,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,560 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:02,608 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-11T07:32:02,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:02,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:32:02,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:02,608 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:02,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:02,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:02,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-11T07:32:02,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-11T07:32:02,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:02,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:32:02,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:02,761 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:02,761 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:02,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:02,769 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310382765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:02,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:02,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:02,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310382767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:02,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,770 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:02,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:02,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310382767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:02,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:02,770 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:02,876 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:32:02,887 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111422318d85ad14bb9842391ef2a18bd7b_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111422318d85ad14bb9842391ef2a18bd7b_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:02,888 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/86d3ac2c63d844b29d01cfd80792f4a0, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:32:02,889 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/86d3ac2c63d844b29d01cfd80792f4a0 is 175, key is test_row_0/A:col10/1731310322078/Put/seqid=0 2024-11-11T07:32:02,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-11T07:32:02,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:02,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:32:02,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:02,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:02,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:02,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:02,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742583_1759 (size=31255) 2024-11-11T07:32:02,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-11T07:32:03,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-11T07:32:03,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:03,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:32:03,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:03,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:03,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:03,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:03,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310383075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:03,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:03,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,077 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310383075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:03,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,077 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:03,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,077 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:03,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310383075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,078 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:03,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-11T07:32:03,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:03,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:32:03,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:03,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:03,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-1 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:03,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:03,322 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=419, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/86d3ac2c63d844b29d01cfd80792f4a0 2024-11-11T07:32:03,353 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/435240eac8c34ab69410fdee04ef7e87 is 50, key is test_row_0/B:col10/1731310322078/Put/seqid=0 2024-11-11T07:32:03,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742584_1760 (size=12301) 2024-11-11T07:32:03,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/435240eac8c34ab69410fdee04ef7e87 2024-11-11T07:32:03,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-11T07:32:03,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:03,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:32:03,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:03,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] handler.RSProcedureHandler(58): pid=177 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:03,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-2 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=177 java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:03,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4177): Remote procedure failed, pid=177 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T07:32:03,384 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/7839132d6e954c0d90ed3c8de3c7e413 is 50, key is test_row_0/C:col10/1731310322078/Put/seqid=0 2024-11-11T07:32:03,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-11T07:32:03,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742585_1761 (size=12301) 2024-11-11T07:32:03,440 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=419 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/7839132d6e954c0d90ed3c8de3c7e413 2024-11-11T07:32:03,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/86d3ac2c63d844b29d01cfd80792f4a0 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/86d3ac2c63d844b29d01cfd80792f4a0 2024-11-11T07:32:03,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/86d3ac2c63d844b29d01cfd80792f4a0, entries=150, sequenceid=419, filesize=30.5 K 2024-11-11T07:32:03,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/435240eac8c34ab69410fdee04ef7e87 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/435240eac8c34ab69410fdee04ef7e87 2024-11-11T07:32:03,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/435240eac8c34ab69410fdee04ef7e87, entries=150, sequenceid=419, filesize=12.0 K 2024-11-11T07:32:03,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/7839132d6e954c0d90ed3c8de3c7e413 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/7839132d6e954c0d90ed3c8de3c7e413 2024-11-11T07:32:03,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/7839132d6e954c0d90ed3c8de3c7e413, entries=150, sequenceid=419, filesize=12.0 K 2024-11-11T07:32:03,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 298c53e334bb69b7b36af68612f3d3c8 in 1049ms, sequenceid=419, compaction requested=false 2024-11-11T07:32:03,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:32:03,535 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33345 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=177 2024-11-11T07:32:03,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:03,535 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-11T07:32:03,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:32:03,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:32:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:32:03,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:03,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bfda0c746ec642ab93f98521dfdde238_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310322426/Put/seqid=0 2024-11-11T07:32:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:03,590 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. as already flushing 2024-11-11T07:32:03,603 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742586_1762 (size=12454) 2024-11-11T07:32:03,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:32:03,617 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111bfda0c746ec642ab93f98521dfdde238_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bfda0c746ec642ab93f98521dfdde238_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:03,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/400e2646361f4dad86793c6c7dec8892, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:32:03,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/400e2646361f4dad86793c6c7dec8892 is 175, key is test_row_0/A:col10/1731310322426/Put/seqid=0 2024-11-11T07:32:03,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742587_1763 (size=31255) 2024-11-11T07:32:03,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310383643, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:03,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,650 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:03,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310383649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:03,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:03,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310383650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:03,755 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310383754, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:03,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,756 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:03,768 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:03,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310383765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,769 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:03,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:03,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310383765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,773 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:03,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:03,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54524 deadline: 1731310383965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:03,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,968 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:03,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:03,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54504 deadline: 1731310383974, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:03,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,975 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:03,985 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-11T07:32:03,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=33345 {}] ipc.CallRunner(138): callId: 179 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54478 deadline: 1731310383984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 , the old value is region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-11T07:32:03,986 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., hostname=ce2cf0244921,33345,1731310155369, seqNum=5 because the exception is null or not the one we care about 2024-11-11T07:32:04,035 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=432, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/400e2646361f4dad86793c6c7dec8892 2024-11-11T07:32:04,044 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/4b0c849363ef444d8b2306f6ea5c8d57 is 50, key is test_row_0/B:col10/1731310322426/Put/seqid=0 2024-11-11T07:32:04,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742588_1764 (size=12301) 2024-11-11T07:32:04,085 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/4b0c849363ef444d8b2306f6ea5c8d57 2024-11-11T07:32:04,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/233fdfb2f18e46dbbd3b3b267d6a7a93 is 50, key is test_row_0/C:col10/1731310322426/Put/seqid=0 2024-11-11T07:32:04,098 INFO [Thread-2860 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2860. 2024-11-11T07:32:04,098 DEBUG [Thread-2860 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:32:04,098 DEBUG [Thread-2860 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,098 DEBUG [Thread-2860 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,098 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:32:04,100 INFO [Thread-2854 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2854. 2024-11-11T07:32:04,100 DEBUG [Thread-2854 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:32:04,100 DEBUG [Thread-2854 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,100 DEBUG [Thread-2854 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,100 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:32:04,101 INFO [Thread-2862 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2862. 2024-11-11T07:32:04,101 DEBUG [Thread-2862 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:32:04,101 DEBUG [Thread-2862 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,101 DEBUG [Thread-2862 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:32:04,103 INFO [Thread-2856 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2856. 2024-11-11T07:32:04,103 DEBUG [Thread-2856 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:32:04,103 DEBUG [Thread-2856 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,104 DEBUG [Thread-2856 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742589_1765 (size=12301) 2024-11-11T07:32:04,104 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=432 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/233fdfb2f18e46dbbd3b3b267d6a7a93 2024-11-11T07:32:04,106 INFO [Thread-2858 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2858. 2024-11-11T07:32:04,106 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:32:04,106 DEBUG [Thread-2858 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:32:04,106 DEBUG [Thread-2858 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,106 DEBUG [Thread-2858 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,107 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:32:04,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/400e2646361f4dad86793c6c7dec8892 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/400e2646361f4dad86793c6c7dec8892 2024-11-11T07:32:04,109 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/400e2646361f4dad86793c6c7dec8892, entries=150, sequenceid=432, filesize=30.5 K 2024-11-11T07:32:04,110 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/4b0c849363ef444d8b2306f6ea5c8d57 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/4b0c849363ef444d8b2306f6ea5c8d57 2024-11-11T07:32:04,112 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/4b0c849363ef444d8b2306f6ea5c8d57, entries=150, sequenceid=432, filesize=12.0 K 2024-11-11T07:32:04,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/233fdfb2f18e46dbbd3b3b267d6a7a93 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/233fdfb2f18e46dbbd3b3b267d6a7a93 2024-11-11T07:32:04,115 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/233fdfb2f18e46dbbd3b3b267d6a7a93, entries=150, sequenceid=432, filesize=12.0 K 2024-11-11T07:32:04,115 INFO [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 298c53e334bb69b7b36af68612f3d3c8 in 580ms, sequenceid=432, compaction requested=true 2024-11-11T07:32:04,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:32:04,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:04,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/ce2cf0244921:0-0 {event_type=RS_FLUSH_REGIONS, pid=177}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=177 2024-11-11T07:32:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster(4169): Remote procedure done, pid=177 2024-11-11T07:32:04,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=177, resume processing ppid=176 2024-11-11T07:32:04,118 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=177, ppid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8170 sec 2024-11-11T07:32:04,119 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=176, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=176, table=TestAcidGuarantees in 1.8220 sec 2024-11-11T07:32:04,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33345 {}] regionserver.HRegion(8855): Flush requested on 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:04,276 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-11T07:32:04,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:32:04,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:04,276 INFO [Thread-2845 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2845. 2024-11-11T07:32:04,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:32:04,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:04,276 DEBUG [Thread-2845 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:32:04,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:32:04,276 DEBUG [Thread-2845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:04,276 DEBUG [Thread-2845 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,276 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:32:04,285 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111100c6249ea5fd480b97de733e73714c88_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_0/A:col10/1731310323638/Put/seqid=0 2024-11-11T07:32:04,286 INFO [Thread-2843 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2843. 2024-11-11T07:32:04,286 DEBUG [Thread-2843 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:32:04,286 DEBUG [Thread-2843 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,286 DEBUG [Thread-2843 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,286 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:32:04,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742590_1766 (size=12454) 2024-11-11T07:32:04,296 INFO [Thread-2851 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2851. 2024-11-11T07:32:04,296 DEBUG [Thread-2851 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:32:04,296 DEBUG [Thread-2851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,296 DEBUG [Thread-2851 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,296 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:32:04,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-11-11T07:32:04,444 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:32:04,566 INFO [Thread-2847 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2847. 2024-11-11T07:32:04,566 DEBUG [Thread-2847 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:32:04,566 DEBUG [Thread-2847 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,567 DEBUG [Thread-2847 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,567 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:32:04,616 INFO [Thread-2849 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2849. 2024-11-11T07:32:04,616 DEBUG [Thread-2849 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-11T07:32:04,616 DEBUG [Thread-2849 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,616 DEBUG [Thread-2849 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,616 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:32:04,616 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-11T07:32:04,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-11T07:32:04,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 111 2024-11-11T07:32:04,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 10 2024-11-11T07:32:04,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 12 2024-11-11T07:32:04,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-11-11T07:32:04,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-11T07:32:04,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2278 2024-11-11T07:32:04,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2267 2024-11-11T07:32:04,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2344 2024-11-11T07:32:04,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2326 2024-11-11T07:32:04,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2246 2024-11-11T07:32:04,617 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-11T07:32:04,617 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T07:32:04,617 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:32:04,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,617 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:04,617 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:32:04,618 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-11T07:32:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-11T07:32:04,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-11T07:32:04,620 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310324620"}]},"ts":"1731310324620"} 2024-11-11T07:32:04,622 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-11T07:32:04,622 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-11T07:32:04,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=179, ppid=178, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-11T07:32:04,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=180, ppid=179, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=298c53e334bb69b7b36af68612f3d3c8, UNASSIGN}] 2024-11-11T07:32:04,624 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=180, ppid=179, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=298c53e334bb69b7b36af68612f3d3c8, UNASSIGN 2024-11-11T07:32:04,627 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=180 updating hbase:meta row=298c53e334bb69b7b36af68612f3d3c8, regionState=CLOSING, regionLocation=ce2cf0244921,33345,1731310155369 2024-11-11T07:32:04,628 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=180, ppid=179, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=298c53e334bb69b7b36af68612f3d3c8, UNASSIGN because future has completed 2024-11-11T07:32:04,628 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-11T07:32:04,629 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=181, ppid=180, state=RUNNABLE, hasLock=false; CloseRegionProcedure 298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369}] 2024-11-11T07:32:04,688 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:32:04,691 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111100c6249ea5fd480b97de733e73714c88_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111100c6249ea5fd480b97de733e73714c88_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:04,691 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/142b6e73c70e4dd68eed89d96b8227f1, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:32:04,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/142b6e73c70e4dd68eed89d96b8227f1 is 175, key is test_row_0/A:col10/1731310323638/Put/seqid=0 2024-11-11T07:32:04,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742591_1767 (size=31255) 2024-11-11T07:32:04,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-11T07:32:04,781 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] handler.UnassignRegionHandler(122): Close 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:04,781 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-11T07:32:04,781 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1722): Closing 298c53e334bb69b7b36af68612f3d3c8, disabling compactions & flushes 2024-11-11T07:32:04,781 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:04,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-11T07:32:05,099 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=452, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/142b6e73c70e4dd68eed89d96b8227f1 2024-11-11T07:32:05,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/0cba1a7a35fc4e118794b0d3f14a1dc3 is 50, key is test_row_0/B:col10/1731310323638/Put/seqid=0 2024-11-11T07:32:05,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742592_1768 (size=12301) 2024-11-11T07:32:05,109 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/0cba1a7a35fc4e118794b0d3f14a1dc3 2024-11-11T07:32:05,114 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/110b21ef51384f579b740a9c0435c417 is 50, key is test_row_0/C:col10/1731310323638/Put/seqid=0 2024-11-11T07:32:05,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742593_1769 (size=12301) 2024-11-11T07:32:05,118 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=452 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/110b21ef51384f579b740a9c0435c417 2024-11-11T07:32:05,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/142b6e73c70e4dd68eed89d96b8227f1 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/142b6e73c70e4dd68eed89d96b8227f1 2024-11-11T07:32:05,124 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/142b6e73c70e4dd68eed89d96b8227f1, entries=150, sequenceid=452, filesize=30.5 K 2024-11-11T07:32:05,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/0cba1a7a35fc4e118794b0d3f14a1dc3 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/0cba1a7a35fc4e118794b0d3f14a1dc3 2024-11-11T07:32:05,127 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/0cba1a7a35fc4e118794b0d3f14a1dc3, entries=150, sequenceid=452, filesize=12.0 K 2024-11-11T07:32:05,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/110b21ef51384f579b740a9c0435c417 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/110b21ef51384f579b740a9c0435c417 2024-11-11T07:32:05,130 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/110b21ef51384f579b740a9c0435c417, entries=150, sequenceid=452, filesize=12.0 K 2024-11-11T07:32:05,131 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=26.84 KB/27480 for 298c53e334bb69b7b36af68612f3d3c8 in 855ms, sequenceid=452, compaction requested=true 2024-11-11T07:32:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 298c53e334bb69b7b36af68612f3d3c8: 2024-11-11T07:32:05,131 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:05,131 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:05,131 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. after waiting 0 ms 2024-11-11T07:32:05,131 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. because compaction request was cancelled 2024-11-11T07:32:05,131 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:05,131 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:A 2024-11-11T07:32:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:A, priority=-2147483648, current under compaction store size is 0 2024-11-11T07:32:05,131 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(2902): Flushing 298c53e334bb69b7b36af68612f3d3c8 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-11T07:32:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:32:05,131 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. because compaction request was cancelled 2024-11-11T07:32:05,131 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:B 2024-11-11T07:32:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:B, priority=-2147483648, current under compaction store size is 0 2024-11-11T07:32:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:32:05,131 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. because compaction request was cancelled 2024-11-11T07:32:05,131 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=A 2024-11-11T07:32:05,131 DEBUG [RS:0;ce2cf0244921:33345-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 298c53e334bb69b7b36af68612f3d3c8:C 2024-11-11T07:32:05,131 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 298c53e334bb69b7b36af68612f3d3c8:C, priority=-2147483648, current under compaction store size is 0 2024-11-11T07:32:05,131 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-11T07:32:05,131 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=B 2024-11-11T07:32:05,131 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:05,131 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 298c53e334bb69b7b36af68612f3d3c8, store=C 2024-11-11T07:32:05,131 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-11T07:32:05,135 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b33a8169e8d54f06b919f790e5bbe255_298c53e334bb69b7b36af68612f3d3c8 is 50, key is test_row_1/A:col10/1731310324615/Put/seqid=0 2024-11-11T07:32:05,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742594_1770 (size=9914) 2024-11-11T07:32:05,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-11T07:32:05,540 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T07:32:05,543 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241111b33a8169e8d54f06b919f790e5bbe255_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b33a8169e8d54f06b919f790e5bbe255_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:05,544 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/cd24455426ec4463aeb15083d28df630, store: [table=TestAcidGuarantees family=A region=298c53e334bb69b7b36af68612f3d3c8] 2024-11-11T07:32:05,544 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/cd24455426ec4463aeb15083d28df630 is 175, key is test_row_1/A:col10/1731310324615/Put/seqid=0 2024-11-11T07:32:05,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742595_1771 (size=22561) 2024-11-11T07:32:05,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-11T07:32:05,948 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=459, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/cd24455426ec4463aeb15083d28df630 2024-11-11T07:32:05,954 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/c8c1583a42d242d890596fa81951043a is 50, key is test_row_1/B:col10/1731310324615/Put/seqid=0 2024-11-11T07:32:05,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742596_1772 (size=9857) 2024-11-11T07:32:06,357 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/c8c1583a42d242d890596fa81951043a 2024-11-11T07:32:06,362 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/91e0ee7e71014fcc8509e1d58106f837 is 50, key is test_row_1/C:col10/1731310324615/Put/seqid=0 2024-11-11T07:32:06,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742597_1773 (size=9857) 2024-11-11T07:32:06,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-11T07:32:06,765 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=459 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/91e0ee7e71014fcc8509e1d58106f837 2024-11-11T07:32:06,768 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/A/cd24455426ec4463aeb15083d28df630 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/cd24455426ec4463aeb15083d28df630 2024-11-11T07:32:06,771 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/cd24455426ec4463aeb15083d28df630, entries=100, sequenceid=459, filesize=22.0 K 2024-11-11T07:32:06,772 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/B/c8c1583a42d242d890596fa81951043a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/c8c1583a42d242d890596fa81951043a 2024-11-11T07:32:06,774 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/c8c1583a42d242d890596fa81951043a, entries=100, sequenceid=459, filesize=9.6 K 2024-11-11T07:32:06,775 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/.tmp/C/91e0ee7e71014fcc8509e1d58106f837 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/91e0ee7e71014fcc8509e1d58106f837 2024-11-11T07:32:06,777 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/91e0ee7e71014fcc8509e1d58106f837, entries=100, sequenceid=459, filesize=9.6 K 2024-11-11T07:32:06,778 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 298c53e334bb69b7b36af68612f3d3c8 in 1647ms, sequenceid=459, compaction requested=true 2024-11-11T07:32:06,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/afc88d82b1944f29b36e51364cfbbb8f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/a82f8dc1a36f471eb3ec0d1b17a3aefa, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/5a1b6fa78a7f4c588db530b9d1f0f757, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/390146f70796429fa23f47f34bfb880f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/8743ba9ca75948219b43bbbabcef7f6b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/e1a218f0c24d40f78ad36d9a52dd54cd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/c559194535114d029ac7b56757081cfd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/c71e3465912b44c887587fc266ab8897, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/73f342402564428c84a9a0f2aea8995c, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/58ee058c7e044058992790a0c370390e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/a325a9b561284378b796fca869cfc474, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/b80270a69adf49c5b42af7ba09fbc8e1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/24b077aaaa5b41559b4fa591c9f42dad, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/da9b7bdfc4f941e4a520e7b2184c91fe, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/8a508b02056e4ceba4e0b3fbe42cd93a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/fa53da60d98b49c9a7b35ed70af4b08f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/ca12b898149746aaa058edc081d25fe1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/481a10558f6b4239aaf32d4b28b3d716, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/59c6d412d24948b6a67b814d3311f011, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/0780849a28b5407e9fcdc001e387eb77, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/e08a465b869449318446afd6ac1acf15, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/6dc704c7618d4837b65cec53a77ed6db, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/95127becff0b458ebfa8178d22b07150, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/f508a24a98a241e3b0efef962d6b1705, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/07a17c28ce674a5ca8d1f64420f6963b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/b6704bc8615544bdbc9ef274d84be5eb] to archive 2024-11-11T07:32:06,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:32:06,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/afc88d82b1944f29b36e51364cfbbb8f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/afc88d82b1944f29b36e51364cfbbb8f 2024-11-11T07:32:06,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/a82f8dc1a36f471eb3ec0d1b17a3aefa to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/a82f8dc1a36f471eb3ec0d1b17a3aefa 2024-11-11T07:32:06,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/5a1b6fa78a7f4c588db530b9d1f0f757 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/5a1b6fa78a7f4c588db530b9d1f0f757 2024-11-11T07:32:06,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/390146f70796429fa23f47f34bfb880f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/390146f70796429fa23f47f34bfb880f 2024-11-11T07:32:06,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/8743ba9ca75948219b43bbbabcef7f6b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/8743ba9ca75948219b43bbbabcef7f6b 2024-11-11T07:32:06,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/e1a218f0c24d40f78ad36d9a52dd54cd to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/e1a218f0c24d40f78ad36d9a52dd54cd 2024-11-11T07:32:06,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/c559194535114d029ac7b56757081cfd to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/c559194535114d029ac7b56757081cfd 2024-11-11T07:32:06,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/c71e3465912b44c887587fc266ab8897 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/c71e3465912b44c887587fc266ab8897 2024-11-11T07:32:06,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/73f342402564428c84a9a0f2aea8995c to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/73f342402564428c84a9a0f2aea8995c 2024-11-11T07:32:06,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/58ee058c7e044058992790a0c370390e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/58ee058c7e044058992790a0c370390e 2024-11-11T07:32:06,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/a325a9b561284378b796fca869cfc474 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/a325a9b561284378b796fca869cfc474 2024-11-11T07:32:06,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/b80270a69adf49c5b42af7ba09fbc8e1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/b80270a69adf49c5b42af7ba09fbc8e1 2024-11-11T07:32:06,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/24b077aaaa5b41559b4fa591c9f42dad to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/24b077aaaa5b41559b4fa591c9f42dad 2024-11-11T07:32:06,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/da9b7bdfc4f941e4a520e7b2184c91fe to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/da9b7bdfc4f941e4a520e7b2184c91fe 2024-11-11T07:32:06,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/8a508b02056e4ceba4e0b3fbe42cd93a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/8a508b02056e4ceba4e0b3fbe42cd93a 2024-11-11T07:32:06,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/fa53da60d98b49c9a7b35ed70af4b08f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/fa53da60d98b49c9a7b35ed70af4b08f 2024-11-11T07:32:06,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/ca12b898149746aaa058edc081d25fe1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/ca12b898149746aaa058edc081d25fe1 2024-11-11T07:32:06,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/481a10558f6b4239aaf32d4b28b3d716 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/481a10558f6b4239aaf32d4b28b3d716 2024-11-11T07:32:06,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/59c6d412d24948b6a67b814d3311f011 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/59c6d412d24948b6a67b814d3311f011 2024-11-11T07:32:06,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/0780849a28b5407e9fcdc001e387eb77 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/0780849a28b5407e9fcdc001e387eb77 2024-11-11T07:32:06,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/e08a465b869449318446afd6ac1acf15 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/e08a465b869449318446afd6ac1acf15 2024-11-11T07:32:06,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/6dc704c7618d4837b65cec53a77ed6db to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/6dc704c7618d4837b65cec53a77ed6db 2024-11-11T07:32:06,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/95127becff0b458ebfa8178d22b07150 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/95127becff0b458ebfa8178d22b07150 2024-11-11T07:32:06,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/f508a24a98a241e3b0efef962d6b1705 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/f508a24a98a241e3b0efef962d6b1705 2024-11-11T07:32:06,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/07a17c28ce674a5ca8d1f64420f6963b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/07a17c28ce674a5ca8d1f64420f6963b 2024-11-11T07:32:06,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/b6704bc8615544bdbc9ef274d84be5eb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/b6704bc8615544bdbc9ef274d84be5eb 2024-11-11T07:32:06,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/283c2be50f47494d9ddc55e4fb267e6f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/d87991db832c473b81feb3aa1b28b34d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/494a54ea89dd4735aa7d45a65aa70e01, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/2a1660a4b3de45278271a2322faee91e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/b884fc5936e14dd18e5ce788f5037858, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/e06d60a9169e4e08a8ab2df24fa63f3b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/e3d621e362324e519d39cdd6a4248bc1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/92b9f5d41bd2466583b12e77a28af41e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/087414cb891846ee9aaf1eadb6fc926a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/01543f79bca6463fbf8b8ecad780bee6, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/4006e153e6fd4ef0926faa0f07064d86, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/990739c8ad6a4b19965c1b90f6d6693d, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/133e9af7e71f45e291ba5ca7db4b3821, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/7e9570293ee546b3bff7f7d7458f1b33, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/948590fb4b4d43baaff061f4581a958f, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/765bb4b958f24fbe8ed673cead4d18c2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/9b528c2aabb24c58b7a2d3f8e7b91fc0, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/f36e4e307fac47758b7dd253fe1ef03a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/02c9412504f4433995a7fb3b1aea3636, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/5c10e1f954334f4481f364931ee9db0e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/32d36182afc847a4b3498ae1d5bc71fe, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/526c59f41a3f435ea1fe8270a310d02b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/a275288531534c29934df4deeff2ddcd, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/65911a4b448b4c0981b506e11b6f93c7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/265549c9798e47f69496698316f0f397, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/cf00181e7c7342a18201b93176b25735] to archive 2024-11-11T07:32:06,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:32:06,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/283c2be50f47494d9ddc55e4fb267e6f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/283c2be50f47494d9ddc55e4fb267e6f 2024-11-11T07:32:06,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/d87991db832c473b81feb3aa1b28b34d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/d87991db832c473b81feb3aa1b28b34d 2024-11-11T07:32:06,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/494a54ea89dd4735aa7d45a65aa70e01 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/494a54ea89dd4735aa7d45a65aa70e01 2024-11-11T07:32:06,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/2a1660a4b3de45278271a2322faee91e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/2a1660a4b3de45278271a2322faee91e 2024-11-11T07:32:06,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/b884fc5936e14dd18e5ce788f5037858 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/b884fc5936e14dd18e5ce788f5037858 2024-11-11T07:32:06,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/e06d60a9169e4e08a8ab2df24fa63f3b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/e06d60a9169e4e08a8ab2df24fa63f3b 2024-11-11T07:32:06,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/e3d621e362324e519d39cdd6a4248bc1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/e3d621e362324e519d39cdd6a4248bc1 2024-11-11T07:32:06,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/92b9f5d41bd2466583b12e77a28af41e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/92b9f5d41bd2466583b12e77a28af41e 2024-11-11T07:32:06,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/087414cb891846ee9aaf1eadb6fc926a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/087414cb891846ee9aaf1eadb6fc926a 2024-11-11T07:32:06,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/01543f79bca6463fbf8b8ecad780bee6 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/01543f79bca6463fbf8b8ecad780bee6 2024-11-11T07:32:06,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/4006e153e6fd4ef0926faa0f07064d86 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/4006e153e6fd4ef0926faa0f07064d86 2024-11-11T07:32:06,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/990739c8ad6a4b19965c1b90f6d6693d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/990739c8ad6a4b19965c1b90f6d6693d 2024-11-11T07:32:06,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/133e9af7e71f45e291ba5ca7db4b3821 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/133e9af7e71f45e291ba5ca7db4b3821 2024-11-11T07:32:06,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/7e9570293ee546b3bff7f7d7458f1b33 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/7e9570293ee546b3bff7f7d7458f1b33 2024-11-11T07:32:06,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/948590fb4b4d43baaff061f4581a958f to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/948590fb4b4d43baaff061f4581a958f 2024-11-11T07:32:06,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/765bb4b958f24fbe8ed673cead4d18c2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/765bb4b958f24fbe8ed673cead4d18c2 2024-11-11T07:32:06,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/9b528c2aabb24c58b7a2d3f8e7b91fc0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/9b528c2aabb24c58b7a2d3f8e7b91fc0 2024-11-11T07:32:06,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/f36e4e307fac47758b7dd253fe1ef03a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/f36e4e307fac47758b7dd253fe1ef03a 2024-11-11T07:32:06,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/02c9412504f4433995a7fb3b1aea3636 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/02c9412504f4433995a7fb3b1aea3636 2024-11-11T07:32:06,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/5c10e1f954334f4481f364931ee9db0e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/5c10e1f954334f4481f364931ee9db0e 2024-11-11T07:32:06,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/32d36182afc847a4b3498ae1d5bc71fe to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/32d36182afc847a4b3498ae1d5bc71fe 2024-11-11T07:32:06,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/526c59f41a3f435ea1fe8270a310d02b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/526c59f41a3f435ea1fe8270a310d02b 2024-11-11T07:32:06,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/a275288531534c29934df4deeff2ddcd to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/a275288531534c29934df4deeff2ddcd 2024-11-11T07:32:06,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/65911a4b448b4c0981b506e11b6f93c7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/65911a4b448b4c0981b506e11b6f93c7 2024-11-11T07:32:06,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/265549c9798e47f69496698316f0f397 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/265549c9798e47f69496698316f0f397 2024-11-11T07:32:06,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/cf00181e7c7342a18201b93176b25735 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/cf00181e7c7342a18201b93176b25735 2024-11-11T07:32:06,832 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/becc423616964b78a4edce00e75917f7, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/0756f8fb796944339a0714f842949054, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/74fa0ce706c54c18911a82d90feacd5e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/e50386f66aff4a868b9a06921d2df151, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/c5bcfb089ed546899cf4ec599cbd3ead, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/994d488d826741dcaabdac9db56047d2, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/b238065298e644e0bbac8a6033ad1947, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/a4fa9388740f42c9a889e9796cd8686b, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/3af171c392424312b9068f947b72c132, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/a1816f36c09a416c90d200fe8b0332b1, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/32c9a0bf7c2041e1a12b66410ebde065, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/3fea9af7ddfd4375975f1bcc9d883ada, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/b6408bd5604a459ab442139768c3ee3e, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/02fdfc3654ce42c0ba136d3f2fa7fd00, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/49171b94b92843769787d77bbc683e0a, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/78951bfd7b7249b49864929ce8d8eb63, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/fa44d48475ce4037aa07f571df8b00ec, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/6cbc96bdf64349ffb9f2a257d41ad538, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/6bb008c6b5174b41947225af326e3f50, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/341a1df16d0d4d9e9f55b0381c9cbed9, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/419b996554614bf2986e70a498dfe474, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/bd70238f0994419eabcd10dbdb414bca, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/8d2f57b9e1404ef3ae48f3b2a42ac3ad, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/c2a6177910d24e3ca47f3b85f76e8aca, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/9be869f71b4c44cea5785cd6a1f72c7d] to archive 2024-11-11T07:32:06,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-11T07:32:06,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/becc423616964b78a4edce00e75917f7 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/becc423616964b78a4edce00e75917f7 2024-11-11T07:32:06,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/0756f8fb796944339a0714f842949054 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/0756f8fb796944339a0714f842949054 2024-11-11T07:32:06,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/74fa0ce706c54c18911a82d90feacd5e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/74fa0ce706c54c18911a82d90feacd5e 2024-11-11T07:32:06,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/e50386f66aff4a868b9a06921d2df151 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/e50386f66aff4a868b9a06921d2df151 2024-11-11T07:32:06,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/c5bcfb089ed546899cf4ec599cbd3ead to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/c5bcfb089ed546899cf4ec599cbd3ead 2024-11-11T07:32:06,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/994d488d826741dcaabdac9db56047d2 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/994d488d826741dcaabdac9db56047d2 2024-11-11T07:32:06,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/b238065298e644e0bbac8a6033ad1947 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/b238065298e644e0bbac8a6033ad1947 2024-11-11T07:32:06,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/a4fa9388740f42c9a889e9796cd8686b to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/a4fa9388740f42c9a889e9796cd8686b 2024-11-11T07:32:06,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/3af171c392424312b9068f947b72c132 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/3af171c392424312b9068f947b72c132 2024-11-11T07:32:06,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/a1816f36c09a416c90d200fe8b0332b1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/a1816f36c09a416c90d200fe8b0332b1 2024-11-11T07:32:06,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/32c9a0bf7c2041e1a12b66410ebde065 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/32c9a0bf7c2041e1a12b66410ebde065 2024-11-11T07:32:06,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/3fea9af7ddfd4375975f1bcc9d883ada to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/3fea9af7ddfd4375975f1bcc9d883ada 2024-11-11T07:32:06,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/b6408bd5604a459ab442139768c3ee3e to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/b6408bd5604a459ab442139768c3ee3e 2024-11-11T07:32:06,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/02fdfc3654ce42c0ba136d3f2fa7fd00 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/02fdfc3654ce42c0ba136d3f2fa7fd00 2024-11-11T07:32:06,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/49171b94b92843769787d77bbc683e0a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/49171b94b92843769787d77bbc683e0a 2024-11-11T07:32:06,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/78951bfd7b7249b49864929ce8d8eb63 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/78951bfd7b7249b49864929ce8d8eb63 2024-11-11T07:32:06,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/fa44d48475ce4037aa07f571df8b00ec to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/fa44d48475ce4037aa07f571df8b00ec 2024-11-11T07:32:06,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/6cbc96bdf64349ffb9f2a257d41ad538 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/6cbc96bdf64349ffb9f2a257d41ad538 2024-11-11T07:32:06,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/6bb008c6b5174b41947225af326e3f50 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/6bb008c6b5174b41947225af326e3f50 2024-11-11T07:32:06,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/341a1df16d0d4d9e9f55b0381c9cbed9 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/341a1df16d0d4d9e9f55b0381c9cbed9 2024-11-11T07:32:06,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/419b996554614bf2986e70a498dfe474 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/419b996554614bf2986e70a498dfe474 2024-11-11T07:32:06,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/bd70238f0994419eabcd10dbdb414bca to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/bd70238f0994419eabcd10dbdb414bca 2024-11-11T07:32:06,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/8d2f57b9e1404ef3ae48f3b2a42ac3ad to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/8d2f57b9e1404ef3ae48f3b2a42ac3ad 2024-11-11T07:32:06,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/c2a6177910d24e3ca47f3b85f76e8aca to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/c2a6177910d24e3ca47f3b85f76e8aca 2024-11-11T07:32:06,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/9be869f71b4c44cea5785cd6a1f72c7d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/9be869f71b4c44cea5785cd6a1f72c7d 2024-11-11T07:32:06,857 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/recovered.edits/462.seqid, newMaxSeqId=462, maxSeqId=4 2024-11-11T07:32:06,857 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. 2024-11-11T07:32:06,857 DEBUG [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] regionserver.HRegion(1676): Region close journal for 298c53e334bb69b7b36af68612f3d3c8: Waiting for close lock at 1731310324781Running coprocessor pre-close hooks at 1731310324781Disabling compacts and flushes for region at 1731310324781Disabling writes for close at 1731310325131 (+350 ms)Obtaining lock to block concurrent updates at 1731310325131Preparing flush snapshotting stores in 298c53e334bb69b7b36af68612f3d3c8 at 1731310325131Finished memstore snapshotting TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731310325131Flushing stores of TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8. at 1731310325132 (+1 ms)Flushing 298c53e334bb69b7b36af68612f3d3c8/A: creating writer at 1731310325132Flushing mob file 298c53e334bb69b7b36af68612f3d3c8/A: appending metadata at 1731310325135 (+3 ms)Flushing mob file 298c53e334bb69b7b36af68612f3d3c8/A: closing flushed file at 1731310325135Flushing 298c53e334bb69b7b36af68612f3d3c8/A: appending metadata at 1731310325544 (+409 ms)Flushing 298c53e334bb69b7b36af68612f3d3c8/A: closing flushed file at 1731310325544Flushing 298c53e334bb69b7b36af68612f3d3c8/B: creating writer at 1731310325951 (+407 ms)Flushing 298c53e334bb69b7b36af68612f3d3c8/B: appending metadata at 1731310325953 (+2 ms)Flushing 298c53e334bb69b7b36af68612f3d3c8/B: closing flushed file at 1731310325953Flushing 298c53e334bb69b7b36af68612f3d3c8/C: creating writer at 1731310326360 (+407 ms)Flushing 298c53e334bb69b7b36af68612f3d3c8/C: appending metadata at 1731310326362 (+2 ms)Flushing 298c53e334bb69b7b36af68612f3d3c8/C: closing flushed file at 1731310326362Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cc28d68: reopening flushed file at 1731310326768 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57818d0a: reopening flushed file at 1731310326771 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f8887bb: reopening flushed file at 1731310326774 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 298c53e334bb69b7b36af68612f3d3c8 in 1647ms, sequenceid=459, compaction requested=true at 1731310326778 (+4 ms)Writing region close event to WAL at 1731310326854 (+76 ms)Running coprocessor post-close hooks at 1731310326857 (+3 ms)Closed at 1731310326857 2024-11-11T07:32:06,859 INFO [RS_CLOSE_REGION-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_REGION, pid=181}] handler.UnassignRegionHandler(157): Closed 298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:06,860 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=180 updating hbase:meta row=298c53e334bb69b7b36af68612f3d3c8, regionState=CLOSED 2024-11-11T07:32:06,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=181, ppid=180, state=RUNNABLE, hasLock=false; CloseRegionProcedure 298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 because future has completed 2024-11-11T07:32:06,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=181, resume processing ppid=180 2024-11-11T07:32:06,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, ppid=180, state=SUCCESS, hasLock=false; CloseRegionProcedure 298c53e334bb69b7b36af68612f3d3c8, server=ce2cf0244921,33345,1731310155369 in 2.2380 sec 2024-11-11T07:32:06,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=180, resume processing ppid=179 2024-11-11T07:32:06,870 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=180, ppid=179, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=298c53e334bb69b7b36af68612f3d3c8, UNASSIGN in 2.2460 sec 2024-11-11T07:32:06,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=179, resume processing ppid=178 2024-11-11T07:32:06,872 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=179, ppid=178, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.2490 sec 2024-11-11T07:32:06,874 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731310326873"}]},"ts":"1731310326873"} 2024-11-11T07:32:06,875 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-11T07:32:06,875 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-11T07:32:06,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=178, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.2570 sec 2024-11-11T07:32:08,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-11-11T07:32:08,774 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:32:08,775 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-11T07:32:08,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] procedure2.ProcedureExecutor(1139): Stored pid=182, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:32:08,776 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=182, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:32:08,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-11T07:32:08,777 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=182, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:32:08,779 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,781 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C, FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/recovered.edits] 2024-11-11T07:32:08,782 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/142b6e73c70e4dd68eed89d96b8227f1 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/142b6e73c70e4dd68eed89d96b8227f1 2024-11-11T07:32:08,783 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/340ebbd80ae043d6b7797b8635a14d8d to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/340ebbd80ae043d6b7797b8635a14d8d 2024-11-11T07:32:08,784 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/400e2646361f4dad86793c6c7dec8892 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/400e2646361f4dad86793c6c7dec8892 2024-11-11T07:32:08,785 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/86d3ac2c63d844b29d01cfd80792f4a0 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/86d3ac2c63d844b29d01cfd80792f4a0 2024-11-11T07:32:08,786 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/cd24455426ec4463aeb15083d28df630 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/A/cd24455426ec4463aeb15083d28df630 2024-11-11T07:32:08,787 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/0cba1a7a35fc4e118794b0d3f14a1dc3 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/0cba1a7a35fc4e118794b0d3f14a1dc3 2024-11-11T07:32:08,788 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/435240eac8c34ab69410fdee04ef7e87 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/435240eac8c34ab69410fdee04ef7e87 2024-11-11T07:32:08,789 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/4ac3c92b7b7343988b02e5fcfac8d951 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/4ac3c92b7b7343988b02e5fcfac8d951 2024-11-11T07:32:08,789 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/4b0c849363ef444d8b2306f6ea5c8d57 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/4b0c849363ef444d8b2306f6ea5c8d57 2024-11-11T07:32:08,790 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/c8c1583a42d242d890596fa81951043a to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/B/c8c1583a42d242d890596fa81951043a 2024-11-11T07:32:08,792 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/110b21ef51384f579b740a9c0435c417 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/110b21ef51384f579b740a9c0435c417 2024-11-11T07:32:08,792 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/233fdfb2f18e46dbbd3b3b267d6a7a93 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/233fdfb2f18e46dbbd3b3b267d6a7a93 2024-11-11T07:32:08,793 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/49a52d5e4cf24963bbd12afd55433bcb to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/49a52d5e4cf24963bbd12afd55433bcb 2024-11-11T07:32:08,794 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/7839132d6e954c0d90ed3c8de3c7e413 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/7839132d6e954c0d90ed3c8de3c7e413 2024-11-11T07:32:08,794 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/91e0ee7e71014fcc8509e1d58106f837 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/C/91e0ee7e71014fcc8509e1d58106f837 2024-11-11T07:32:08,796 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/recovered.edits/462.seqid to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8/recovered.edits/462.seqid 2024-11-11T07:32:08,796 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/default/TestAcidGuarantees/298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,797 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-11T07:32:08,797 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T07:32:08,797 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-11T07:32:08,799 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111100c6249ea5fd480b97de733e73714c88_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111100c6249ea5fd480b97de733e73714c88_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,800 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110ca150a8332b436b94f13ded9d834b17_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411110ca150a8332b436b94f13ded9d834b17_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,801 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111187250d69f1d4332ab3354e835bb3152_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111187250d69f1d4332ab3354e835bb3152_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,802 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111124fd9da94d9c46fe8e4ab4aa074c855e_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111124fd9da94d9c46fe8e4ab4aa074c855e_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,802 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111128288b6249dd4b41b9264f9669e2253b_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111128288b6249dd4b41b9264f9669e2253b_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,803 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111422318d85ad14bb9842391ef2a18bd7b_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111422318d85ad14bb9842391ef2a18bd7b_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,804 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114718999e3ebf4ee3bf836daebfdd5b17_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411114718999e3ebf4ee3bf836daebfdd5b17_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,804 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118199e9c9ff7041e491d2677d905d4ae3_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411118199e9c9ff7041e491d2677d905d4ae3_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,805 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119d09428884ea4289a8d40e69397ab028_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411119d09428884ea4289a8d40e69397ab028_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,806 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b109a838f46348c28bebf6e020f16740_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b109a838f46348c28bebf6e020f16740_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,807 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b16b18cde6bd4f59aeedfced5be5c7c2_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b16b18cde6bd4f59aeedfced5be5c7c2_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,808 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b33a8169e8d54f06b919f790e5bbe255_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b33a8169e8d54f06b919f790e5bbe255_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,809 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b634774760f847719abd13ae8a5d7afa_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111b634774760f847719abd13ae8a5d7afa_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,810 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bcdb64b504a740c6b92c6bd5f4f2b42a_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bcdb64b504a740c6b92c6bd5f4f2b42a_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,810 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bfda0c746ec642ab93f98521dfdde238_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111bfda0c746ec642ab93f98521dfdde238_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,811 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d637c7a3bc754cbdb51218eaf9c4b608_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111d637c7a3bc754cbdb51218eaf9c4b608_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,812 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111eaa02b3273ad43da8ec73cc5701ea77d_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111eaa02b3273ad43da8ec73cc5701ea77d_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,813 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f253823c12a44df7929f2e5b0e5fc8f1_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f253823c12a44df7929f2e5b0e5fc8f1_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,813 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f470154db25f411a9465b5ed814c27df_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f470154db25f411a9465b5ed814c27df_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,814 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f6df429a13ed4280bf1defe2e828e96d_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111f6df429a13ed4280bf1defe2e828e96d_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,815 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fbdffd5b6910472484fafb4d797fe6e8_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fbdffd5b6910472484fafb4d797fe6e8_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,815 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fcb47b7d56064ec38aefa27acdcf0849_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fcb47b7d56064ec38aefa27acdcf0849_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,816 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fdf8b231b0d044009af732fdf3b5c6b4_298c53e334bb69b7b36af68612f3d3c8 to hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241111fdf8b231b0d044009af732fdf3b5c6b4_298c53e334bb69b7b36af68612f3d3c8 2024-11-11T07:32:08,816 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-11T07:32:08,819 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=182, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:32:08,821 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-11T07:32:08,823 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-11T07:32:08,824 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=182, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:32:08,824 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-11T07:32:08,824 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731310328824"}]},"ts":"9223372036854775807"} 2024-11-11T07:32:08,826 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-11T07:32:08,826 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 298c53e334bb69b7b36af68612f3d3c8, NAME => 'TestAcidGuarantees,,1731310301927.298c53e334bb69b7b36af68612f3d3c8.', STARTKEY => '', ENDKEY => ''}] 2024-11-11T07:32:08,826 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-11T07:32:08,826 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731310328826"}]},"ts":"9223372036854775807"} 2024-11-11T07:32:08,828 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-11T07:32:08,829 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=182, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-11T07:32:08,830 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 54 msec 2024-11-11T07:32:08,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33029 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=182 2024-11-11T07:32:08,885 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-11T07:32:08,889 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-11T07:32:08,897 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=252 (was 250) - Thread LEAK? -, OpenFileDescriptor=457 (was 447) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=840 (was 872), ProcessCount=11 (was 11), AvailableMemoryMB=2399 (was 2409) 2024-11-11T07:32:08,897 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-11T07:32:08,897 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T07:32:08,897 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:32:08,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:08,897 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:08,897 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-11T07:32:08,898 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T07:32:08,898 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=958468940, stopped=false 2024-11-11T07:32:08,898 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=ce2cf0244921,33029,1731310154427 2024-11-11T07:32:08,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T07:32:08,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:32:08,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T07:32:08,956 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:32:08,957 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T07:32:08,957 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T07:32:08,957 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T07:32:08,958 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-11T07:32:08,958 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:32:08,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:08,959 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'ce2cf0244921,33345,1731310155369' ***** 2024-11-11T07:32:08,959 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-11T07:32:08,959 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T07:32:08,959 INFO [RS:0;ce2cf0244921:33345 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T07:32:08,959 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-11T07:32:08,959 INFO [RS:0;ce2cf0244921:33345 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T07:32:08,960 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(959): stopping server ce2cf0244921,33345,1731310155369 2024-11-11T07:32:08,960 INFO [RS:0;ce2cf0244921:33345 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T07:32:08,960 INFO [RS:0;ce2cf0244921:33345 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;ce2cf0244921:33345. 2024-11-11T07:32:08,960 DEBUG [RS:0;ce2cf0244921:33345 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T07:32:08,960 DEBUG [RS:0;ce2cf0244921:33345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:08,960 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T07:32:08,960 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T07:32:08,960 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T07:32:08,960 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-11T07:32:08,961 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-11T07:32:08,961 DEBUG [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-11T07:32:08,961 DEBUG [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T07:32:08,961 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-11T07:32:08,961 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-11T07:32:08,961 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-11T07:32:08,961 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T07:32:08,961 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T07:32:08,961 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-11T07:32:08,969 INFO [regionserver/ce2cf0244921:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T07:32:08,969 INFO [regionserver/ce2cf0244921:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T07:32:08,987 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/.tmp/info/cd629777835a4eb88a5cb939c1023bcc is 95, key is TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1./info:/1731310187154/DeleteFamily/seqid=0 2024-11-11T07:32:08,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742598_1774 (size=6514) 2024-11-11T07:32:09,011 INFO [regionserver/ce2cf0244921:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T07:32:09,161 DEBUG [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T07:32:09,361 DEBUG [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T07:32:09,390 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/.tmp/info/cd629777835a4eb88a5cb939c1023bcc 2024-11-11T07:32:09,412 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/.tmp/ns/0c31bdf53db74c2f9fc928f91ea7024a is 93, key is TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1./ns:/1731310187068/DeleteFamily/seqid=0 2024-11-11T07:32:09,415 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742599_1775 (size=5906) 2024-11-11T07:32:09,562 DEBUG [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T07:32:09,762 DEBUG [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T07:32:09,815 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/.tmp/ns/0c31bdf53db74c2f9fc928f91ea7024a 2024-11-11T07:32:09,885 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/.tmp/rep_barrier/2d9786b968aa4954aa4e8f277613b362 is 102, key is TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1./rep_barrier:/1731310187068/DeleteFamily/seqid=0 2024-11-11T07:32:09,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742600_1776 (size=6025) 2024-11-11T07:32:09,962 DEBUG [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T07:32:10,162 DEBUG [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-11T07:32:10,288 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/.tmp/rep_barrier/2d9786b968aa4954aa4e8f277613b362 2024-11-11T07:32:10,309 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/.tmp/table/edd5384e63f348bcb830868d871143df is 96, key is TestAcidGuarantees,,1731310159107.6756dec3886a521f131ee738c3465dc1./table:/1731310187068/DeleteFamily/seqid=0 2024-11-11T07:32:10,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742601_1777 (size=5936) 2024-11-11T07:32:10,312 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/.tmp/table/edd5384e63f348bcb830868d871143df 2024-11-11T07:32:10,315 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/.tmp/info/cd629777835a4eb88a5cb939c1023bcc as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/info/cd629777835a4eb88a5cb939c1023bcc 2024-11-11T07:32:10,318 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/info/cd629777835a4eb88a5cb939c1023bcc, entries=12, sequenceid=90, filesize=6.4 K 2024-11-11T07:32:10,319 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/.tmp/ns/0c31bdf53db74c2f9fc928f91ea7024a as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/ns/0c31bdf53db74c2f9fc928f91ea7024a 2024-11-11T07:32:10,321 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/ns/0c31bdf53db74c2f9fc928f91ea7024a, entries=8, sequenceid=90, filesize=5.8 K 2024-11-11T07:32:10,322 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/.tmp/rep_barrier/2d9786b968aa4954aa4e8f277613b362 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/rep_barrier/2d9786b968aa4954aa4e8f277613b362 2024-11-11T07:32:10,324 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/rep_barrier/2d9786b968aa4954aa4e8f277613b362, entries=6, sequenceid=90, filesize=5.9 K 2024-11-11T07:32:10,325 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/.tmp/table/edd5384e63f348bcb830868d871143df as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/table/edd5384e63f348bcb830868d871143df 2024-11-11T07:32:10,328 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/table/edd5384e63f348bcb830868d871143df, entries=7, sequenceid=90, filesize=5.8 K 2024-11-11T07:32:10,328 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1367ms, sequenceid=90, compaction requested=false 2024-11-11T07:32:10,333 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-11T07:32:10,334 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T07:32:10,334 INFO [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-11T07:32:10,334 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731310328961Running coprocessor pre-close hooks at 1731310328961Disabling compacts and flushes for region at 1731310328961Disabling writes for close at 1731310328961Obtaining lock to block concurrent updates at 1731310328961Preparing flush snapshotting stores in 1588230740 at 1731310328961Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731310328962 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731310328962Flushing 1588230740/info: creating writer at 1731310328963 (+1 ms)Flushing 1588230740/info: appending metadata at 1731310328987 (+24 ms)Flushing 1588230740/info: closing flushed file at 1731310328987Flushing 1588230740/ns: creating writer at 1731310329394 (+407 ms)Flushing 1588230740/ns: appending metadata at 1731310329412 (+18 ms)Flushing 1588230740/ns: closing flushed file at 1731310329412Flushing 1588230740/rep_barrier: creating writer at 1731310329867 (+455 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731310329885 (+18 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731310329885Flushing 1588230740/table: creating writer at 1731310330291 (+406 ms)Flushing 1588230740/table: appending metadata at 1731310330309 (+18 ms)Flushing 1588230740/table: closing flushed file at 1731310330309Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69a16913: reopening flushed file at 1731310330315 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@194e27e: reopening flushed file at 1731310330318 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@293ec625: reopening flushed file at 1731310330321 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1585054e: reopening flushed file at 1731310330325 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1367ms, sequenceid=90, compaction requested=false at 1731310330328 (+3 ms)Writing region close event to WAL at 1731310330331 (+3 ms)Running coprocessor post-close hooks at 1731310330333 (+2 ms)Closed at 1731310330334 (+1 ms) 2024-11-11T07:32:10,334 DEBUG [RS_CLOSE_META-regionserver/ce2cf0244921:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T07:32:10,363 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(976): stopping server ce2cf0244921,33345,1731310155369; all regions closed. 2024-11-11T07:32:10,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741834_1010 (size=25344) 2024-11-11T07:32:10,369 DEBUG [RS:0;ce2cf0244921:33345 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/oldWALs 2024-11-11T07:32:10,370 INFO [RS:0;ce2cf0244921:33345 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ce2cf0244921%2C33345%2C1731310155369.meta:.meta(num 1731310158521) 2024-11-11T07:32:10,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741833_1009 (size=19020476) 2024-11-11T07:32:10,376 DEBUG [RS:0;ce2cf0244921:33345 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/oldWALs 2024-11-11T07:32:10,376 INFO [RS:0;ce2cf0244921:33345 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL ce2cf0244921%2C33345%2C1731310155369:(num 1731310158056) 2024-11-11T07:32:10,377 DEBUG [RS:0;ce2cf0244921:33345 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T07:32:10,377 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T07:32:10,377 INFO [RS:0;ce2cf0244921:33345 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T07:32:10,377 INFO [RS:0;ce2cf0244921:33345 {}] hbase.ChoreService(370): Chore service for: regionserver/ce2cf0244921:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-11T07:32:10,377 INFO [RS:0;ce2cf0244921:33345 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T07:32:10,377 INFO [regionserver/ce2cf0244921:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T07:32:10,378 INFO [RS:0;ce2cf0244921:33345 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33345 2024-11-11T07:32:10,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T07:32:10,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/ce2cf0244921,33345,1731310155369 2024-11-11T07:32:10,391 INFO [RS:0;ce2cf0244921:33345 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T07:32:10,393 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [ce2cf0244921,33345,1731310155369] 2024-11-11T07:32:10,408 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/ce2cf0244921,33345,1731310155369 already deleted, retry=false 2024-11-11T07:32:10,408 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; ce2cf0244921,33345,1731310155369 expired; onlineServers=0 2024-11-11T07:32:10,408 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'ce2cf0244921,33029,1731310154427' ***** 2024-11-11T07:32:10,408 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T07:32:10,409 INFO [M:0;ce2cf0244921:33029 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-11T07:32:10,409 INFO [M:0;ce2cf0244921:33029 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-11T07:32:10,409 DEBUG [M:0;ce2cf0244921:33029 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T07:32:10,409 DEBUG [M:0;ce2cf0244921:33029 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T07:32:10,409 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T07:32:10,409 DEBUG [master/ce2cf0244921:0:becomeActiveMaster-HFileCleaner.large.0-1731310157676 {}] cleaner.HFileCleaner(306): Exit Thread[master/ce2cf0244921:0:becomeActiveMaster-HFileCleaner.large.0-1731310157676,5,FailOnTimeoutGroup] 2024-11-11T07:32:10,409 DEBUG [master/ce2cf0244921:0:becomeActiveMaster-HFileCleaner.small.0-1731310157678 {}] cleaner.HFileCleaner(306): Exit Thread[master/ce2cf0244921:0:becomeActiveMaster-HFileCleaner.small.0-1731310157678,5,FailOnTimeoutGroup] 2024-11-11T07:32:10,410 INFO [M:0;ce2cf0244921:33029 {}] hbase.ChoreService(370): Chore service for: master/ce2cf0244921:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-11T07:32:10,410 INFO [M:0;ce2cf0244921:33029 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-11T07:32:10,410 DEBUG [M:0;ce2cf0244921:33029 {}] master.HMaster(1795): Stopping service threads 2024-11-11T07:32:10,410 INFO [M:0;ce2cf0244921:33029 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T07:32:10,410 INFO [M:0;ce2cf0244921:33029 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-11T07:32:10,410 ERROR [M:0;ce2cf0244921:33029 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-5,5,PEWorkerGroup] Thread[IPC Client (1102635514) connection to localhost/127.0.0.1:35677 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:35677,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-11T07:32:10,411 INFO [M:0;ce2cf0244921:33029 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T07:32:10,411 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T07:32:10,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T07:32:10,416 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T07:32:10,416 DEBUG [M:0;ce2cf0244921:33029 {}] zookeeper.ZKUtil(347): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T07:32:10,416 WARN [M:0;ce2cf0244921:33029 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T07:32:10,417 INFO [M:0;ce2cf0244921:33029 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/.lastflushedseqids 2024-11-11T07:32:10,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742602_1778 (size=72) 2024-11-11T07:32:10,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T07:32:10,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33345-0x1012a1601840001, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T07:32:10,501 INFO [RS:0;ce2cf0244921:33345 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T07:32:10,501 INFO [RS:0;ce2cf0244921:33345 {}] regionserver.HRegionServer(1031): Exiting; stopping=ce2cf0244921,33345,1731310155369; zookeeper connection closed. 2024-11-11T07:32:10,501 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1beb939c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1beb939c 2024-11-11T07:32:10,502 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T07:32:10,828 INFO [M:0;ce2cf0244921:33029 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-11T07:32:10,828 INFO [M:0;ce2cf0244921:33029 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T07:32:10,828 DEBUG [M:0;ce2cf0244921:33029 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T07:32:10,828 INFO [M:0;ce2cf0244921:33029 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T07:32:10,828 DEBUG [M:0;ce2cf0244921:33029 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T07:32:10,828 DEBUG [M:0;ce2cf0244921:33029 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T07:32:10,828 DEBUG [M:0;ce2cf0244921:33029 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T07:32:10,828 INFO [M:0;ce2cf0244921:33029 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=824.72 KB heapSize=1013.70 KB 2024-11-11T07:32:10,848 DEBUG [M:0;ce2cf0244921:33029 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/39d18a7688d340b3830304ed3a05cd2b is 82, key is hbase:meta,,1/info:regioninfo/1731310158629/Put/seqid=0 2024-11-11T07:32:10,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742603_1779 (size=5672) 2024-11-11T07:32:11,251 INFO [M:0;ce2cf0244921:33029 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2341 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/39d18a7688d340b3830304ed3a05cd2b 2024-11-11T07:32:11,275 DEBUG [M:0;ce2cf0244921:33029 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3422e4eb71304c37b91b941ec7977729 is 2479, key is \x00\x00\x00\x00\x00\x00\x00#/proc:d/1731310190497/Put/seqid=0 2024-11-11T07:32:11,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742604_1780 (size=47443) 2024-11-11T07:32:11,284 INFO [M:0;ce2cf0244921:33029 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=824.16 KB at sequenceid=2341 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3422e4eb71304c37b91b941ec7977729 2024-11-11T07:32:11,287 INFO [M:0;ce2cf0244921:33029 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3422e4eb71304c37b91b941ec7977729 2024-11-11T07:32:11,307 DEBUG [M:0;ce2cf0244921:33029 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b1734b3a0b0d4a7b8c98748b7d9705ab is 69, key is ce2cf0244921,33345,1731310155369/rs:state/1731310157747/Put/seqid=0 2024-11-11T07:32:11,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073742605_1781 (size=5156) 2024-11-11T07:32:11,315 INFO [M:0;ce2cf0244921:33029 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2341 (bloomFilter=true), to=hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b1734b3a0b0d4a7b8c98748b7d9705ab 2024-11-11T07:32:11,319 DEBUG [M:0;ce2cf0244921:33029 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/39d18a7688d340b3830304ed3a05cd2b as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/39d18a7688d340b3830304ed3a05cd2b 2024-11-11T07:32:11,322 INFO [M:0;ce2cf0244921:33029 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/39d18a7688d340b3830304ed3a05cd2b, entries=8, sequenceid=2341, filesize=5.5 K 2024-11-11T07:32:11,323 DEBUG [M:0;ce2cf0244921:33029 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3422e4eb71304c37b91b941ec7977729 as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3422e4eb71304c37b91b941ec7977729 2024-11-11T07:32:11,326 INFO [M:0;ce2cf0244921:33029 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3422e4eb71304c37b91b941ec7977729 2024-11-11T07:32:11,326 INFO [M:0;ce2cf0244921:33029 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3422e4eb71304c37b91b941ec7977729, entries=182, sequenceid=2341, filesize=46.3 K 2024-11-11T07:32:11,327 DEBUG [M:0;ce2cf0244921:33029 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/b1734b3a0b0d4a7b8c98748b7d9705ab as hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b1734b3a0b0d4a7b8c98748b7d9705ab 2024-11-11T07:32:11,330 INFO [M:0;ce2cf0244921:33029 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:35677/user/jenkins/test-data/28367437-4d03-1d6d-c865-3c6416571f8e/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/b1734b3a0b0d4a7b8c98748b7d9705ab, entries=1, sequenceid=2341, filesize=5.0 K 2024-11-11T07:32:11,331 INFO [M:0;ce2cf0244921:33029 {}] regionserver.HRegion(3140): Finished flush of dataSize ~824.72 KB/844511, heapSize ~1013.41 KB/1037728, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 503ms, sequenceid=2341, compaction requested=false 2024-11-11T07:32:11,332 INFO [M:0;ce2cf0244921:33029 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T07:32:11,332 DEBUG [M:0;ce2cf0244921:33029 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731310330828Disabling compacts and flushes for region at 1731310330828Disabling writes for close at 1731310330828Obtaining lock to block concurrent updates at 1731310330828Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731310330828Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=844511, getHeapSize=1037968, getOffHeapSize=0, getCellsCount=2548 at 1731310330828Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731310330829 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731310330829Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731310330847 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731310330847Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731310331254 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731310331275 (+21 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731310331275Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731310331288 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731310331306 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731310331306Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16d4cba4: reopening flushed file at 1731310331318 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ce5f3fd: reopening flushed file at 1731310331323 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3bd7bddb: reopening flushed file at 1731310331326 (+3 ms)Finished flush of dataSize ~824.72 KB/844511, heapSize ~1013.41 KB/1037728, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 503ms, sequenceid=2341, compaction requested=false at 1731310331331 (+5 ms)Writing region close event to WAL at 1731310331332 (+1 ms)Closed at 1731310331332 2024-11-11T07:32:11,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34017 is added to blk_1073741830_1006 (size=997923) 2024-11-11T07:32:11,336 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-11T07:32:11,336 INFO [M:0;ce2cf0244921:33029 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-11T07:32:11,336 INFO [M:0;ce2cf0244921:33029 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33029 2024-11-11T07:32:11,336 INFO [M:0;ce2cf0244921:33029 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-11T07:32:11,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T07:32:11,489 INFO [M:0;ce2cf0244921:33029 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-11T07:32:11,489 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33029-0x1012a1601840000, quorum=127.0.0.1:52366, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T07:32:11,495 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4269bd31{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T07:32:11,497 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@353d64f2{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T07:32:11,497 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T07:32:11,497 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@435f9f96{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T07:32:11,497 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4fd7fa7c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/hadoop.log.dir/,STOPPED} 2024-11-11T07:32:11,500 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T07:32:11,500 WARN [BP-1596547718-172.17.0.2-1731310150087 heartbeating to localhost/127.0.0.1:35677 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T07:32:11,500 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T07:32:11,500 WARN [BP-1596547718-172.17.0.2-1731310150087 heartbeating to localhost/127.0.0.1:35677 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1596547718-172.17.0.2-1731310150087 (Datanode Uuid 19ca5789-f699-41a7-beb6-23556689d733) service to localhost/127.0.0.1:35677 2024-11-11T07:32:11,504 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/cluster_43578dcd-0ad0-2e50-c990-4483826cf543/data/data1/current/BP-1596547718-172.17.0.2-1731310150087 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T07:32:11,504 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/cluster_43578dcd-0ad0-2e50-c990-4483826cf543/data/data2/current/BP-1596547718-172.17.0.2-1731310150087 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T07:32:11,505 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T07:32:11,513 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4ded99d3{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T07:32:11,514 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@557285fa{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T07:32:11,514 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T07:32:11,514 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@93c62df{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T07:32:11,515 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b23711f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/3439d95e-3149-4203-0781-54a2abcd1ee0/hadoop.log.dir/,STOPPED} 2024-11-11T07:32:11,539 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-11T07:32:11,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down